Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
f805496e2f
commit
2857ba3ce5
|
@ -1,8 +1,5 @@
|
|||
---
|
||||
Naming/HeredocDelimiterNaming:
|
||||
# Offense count: 388
|
||||
# Temporarily disabled due to too many offenses
|
||||
Enabled: false
|
||||
Exclude:
|
||||
- 'app/models/ci/build_trace_chunks/redis_base.rb'
|
||||
- 'app/models/concerns/counter_attribute.rb'
|
||||
|
@ -31,6 +28,7 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'ee/spec/services/security/security_orchestration_policies/policy_commit_service_spec.rb'
|
||||
- 'ee/spec/support/helpers/ee/ldap_helpers.rb'
|
||||
- 'ee/spec/tasks/gitlab/elastic_rake_spec.rb'
|
||||
- 'lib/api/metadata.rb'
|
||||
- 'lib/api/version.rb'
|
||||
- 'lib/backup/helper.rb'
|
||||
- 'lib/feature/shared.rb'
|
||||
|
@ -46,14 +44,14 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'lib/tasks/gitlab/docs/compile_deprecations.rake'
|
||||
- 'lib/tasks/gitlab/password.rake'
|
||||
- 'qa/qa/scenario/test/sanity/selectors.rb'
|
||||
- 'qa/qa/service/docker_run/gitlab_runner.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/3_create/web_ide/web_terminal_spec.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb'
|
||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb'
|
||||
- 'rubocop/cop/database/multiple_databases.rb'
|
||||
- 'rubocop/cop/database/rescue_query_canceled.rb'
|
||||
- 'rubocop/cop/database/rescue_statement_timeout.rb'
|
||||
- 'rubocop/cop/default_scope.rb'
|
||||
- 'rubocop/cop/file_decompression.rb'
|
||||
- 'rubocop/cop/gitlab/httparty.rb'
|
||||
|
@ -61,6 +59,7 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'rubocop/cop/gitlab/module_with_instance_variables.rb'
|
||||
- 'rubocop/cop/gitlab/predicate_memoization.rb'
|
||||
- 'spec/controllers/projects/pipelines_controller_spec.rb'
|
||||
- 'spec/db/docs_spec.rb'
|
||||
- 'spec/deprecation_toolkit_env.rb'
|
||||
- 'spec/factories/packages/debian/distribution.rb'
|
||||
- 'spec/factories/packages/debian/file_metadatum.rb'
|
||||
|
@ -68,6 +67,7 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'spec/features/task_lists_spec.rb'
|
||||
- 'spec/helpers/markup_helper_spec.rb'
|
||||
- 'spec/initializers/100_patch_omniauth_oauth2_spec.rb'
|
||||
- 'spec/initializers/net_http_response_patch_spec.rb'
|
||||
- 'spec/initializers/rack_multipart_patch_spec.rb'
|
||||
- 'spec/initializers/secret_token_spec.rb'
|
||||
- 'spec/initializers/validate_database_config_spec.rb'
|
||||
|
@ -82,6 +82,7 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'spec/lib/gitlab/ci/yaml_processor_spec.rb'
|
||||
- 'spec/lib/gitlab/cluster/mixins/puma_cluster_spec.rb'
|
||||
- 'spec/lib/gitlab/conflict/file_collection_spec.rb'
|
||||
- 'spec/lib/gitlab/database/loose_foreign_keys_spec.rb'
|
||||
- 'spec/lib/gitlab/diff/file_spec.rb'
|
||||
- 'spec/lib/gitlab/diff/pair_selector_spec.rb'
|
||||
- 'spec/lib/gitlab/diff/parser_spec.rb'
|
||||
|
@ -131,10 +132,10 @@ Naming/HeredocDelimiterNaming:
|
|||
- 'spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb'
|
||||
- 'spec/services/google_cloud/generate_pipeline_service_spec.rb'
|
||||
- 'spec/services/task_list_toggle_service_spec.rb'
|
||||
- 'spec/support/helpers/seed_helper.rb'
|
||||
- 'spec/support/helpers/stub_object_storage.rb'
|
||||
- 'spec/support/shared_examples/helm_commands_shared_examples.rb'
|
||||
- 'spec/support/shared_examples/models/taskable_shared_examples.rb'
|
||||
- 'spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb'
|
||||
- 'spec/support/test_reports/test_reports_helper.rb'
|
||||
- 'spec/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences_rake_spec.rb'
|
||||
- 'spec/workers/post_receive_spec.rb'
|
||||
|
|
|
@ -1 +1 @@
|
|||
3d769a33712796de113fe07647597a63d762c305
|
||||
993d944ebeea3e0ec8157481f125f9c70161ae4a
|
||||
|
|
4
Gemfile
4
Gemfile
|
@ -78,7 +78,7 @@ gem 'u2f', '~> 0.2.1'
|
|||
gem 'validates_hostname', '~> 1.0.11'
|
||||
gem 'rubyzip', '~> 2.3.2', require: 'zip'
|
||||
# GitLab Pages letsencrypt support
|
||||
gem 'acme-client', '~> 2.0', '>= 2.0.9'
|
||||
gem 'acme-client', '~> 2.0'
|
||||
|
||||
# Browser detection
|
||||
gem 'browser', '~> 4.2'
|
||||
|
@ -435,6 +435,8 @@ group :test do
|
|||
gem 'capybara-screenshot', '~> 1.0.22'
|
||||
gem 'selenium-webdriver', '~> 3.142'
|
||||
|
||||
gem 'graphlyte', '~> 1.0.0'
|
||||
|
||||
gem 'shoulda-matchers', '~> 5.1.0', require: false
|
||||
gem 'email_spec', '~> 2.2.0'
|
||||
gem 'webmock', '~> 3.9.1'
|
||||
|
|
|
@ -43,8 +43,9 @@ GEM
|
|||
remote: https://rubygems.org/
|
||||
specs:
|
||||
RedCloth (4.3.2)
|
||||
acme-client (2.0.9)
|
||||
faraday (>= 0.17, < 2.0.0)
|
||||
acme-client (2.0.11)
|
||||
faraday (>= 1.0, < 3.0.0)
|
||||
faraday-retry (~> 1.0)
|
||||
actioncable (6.1.6.1)
|
||||
actionpack (= 6.1.6.1)
|
||||
activesupport (= 6.1.6.1)
|
||||
|
@ -615,6 +616,7 @@ GEM
|
|||
faraday (>= 1.0)
|
||||
faraday_middleware
|
||||
graphql-client
|
||||
graphlyte (1.0.0)
|
||||
graphql (1.13.12)
|
||||
graphql-client (0.17.0)
|
||||
activesupport (>= 3.0)
|
||||
|
@ -1481,7 +1483,7 @@ PLATFORMS
|
|||
|
||||
DEPENDENCIES
|
||||
RedCloth (~> 4.3.2)
|
||||
acme-client (~> 2.0, >= 2.0.9)
|
||||
acme-client (~> 2.0)
|
||||
activerecord-explain-analyze (~> 0.1)
|
||||
acts-as-taggable-on (~> 9.0)
|
||||
addressable (~> 2.8)
|
||||
|
@ -1593,6 +1595,7 @@ DEPENDENCIES
|
|||
grape_logging (~> 1.8)
|
||||
graphiql-rails (~> 1.8)
|
||||
graphlient (~> 0.5.0)
|
||||
graphlyte (~> 1.0.0)
|
||||
graphql (~> 1.13.12)
|
||||
graphql-docs (~> 2.1.0)
|
||||
grpc (~> 1.42.0)
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
fragment RunnerFieldsShared on CiRunner {
|
||||
__typename
|
||||
id
|
||||
shortSha
|
||||
runnerType
|
||||
|
|
|
@ -31,7 +31,6 @@ query getAllRunners(
|
|||
editAdminUrl
|
||||
}
|
||||
pageInfo {
|
||||
__typename
|
||||
...PageInfo
|
||||
}
|
||||
}
|
||||
|
|
|
@ -36,7 +36,6 @@ query getGroupRunners(
|
|||
}
|
||||
}
|
||||
pageInfo {
|
||||
__typename
|
||||
...PageInfo
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
fragment ListItemShared on CiRunner {
|
||||
__typename
|
||||
id
|
||||
description
|
||||
runnerType
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
fragment RunnerDetailsShared on CiRunner {
|
||||
__typename
|
||||
id
|
||||
shortSha
|
||||
runnerType
|
||||
|
|
|
@ -41,7 +41,7 @@
|
|||
= n_('%s additional commit has been omitted to prevent performance issues.', '%s additional commits have been omitted to prevent performance issues.', hidden) % number_with_delimiter(hidden)
|
||||
|
||||
- if can_update_merge_request && context_commits&.empty?
|
||||
= render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5', data: { context_commits_empty: 'true' } }) do
|
||||
= render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5 add-review-item-modal-trigger', data: { context_commits_empty: 'true' } }) do
|
||||
= _('Add previously merged commits')
|
||||
|
||||
- if commits.size == 0 && context_commits.nil?
|
||||
|
|
|
@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/364101
|
|||
milestone: '15.1'
|
||||
type: development
|
||||
group: group::source code
|
||||
default_enabled: false
|
||||
default_enabled: true
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class IndexPersonalAccessTokensOnIdAndCreatedAt < Gitlab::Database::Migration[2.0]
|
||||
INDEX_NAME = 'index_personal_access_tokens_on_id_and_created_at'
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
add_concurrent_index :personal_access_tokens, [:id, :created_at], name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
remove_concurrent_index_by_name :personal_access_tokens, INDEX_NAME
|
||||
end
|
||||
end
|
|
@ -0,0 +1 @@
|
|||
b0499c9b4cf3f39eec49dc7def7eaf8f1bbd03f2a34ba9eefa8440a109672136
|
|
@ -29251,6 +29251,8 @@ CREATE INDEX index_path_locks_on_user_id ON path_locks USING btree (user_id);
|
|||
|
||||
CREATE INDEX index_pe_approval_rules_on_required_approvals_and_created_at ON protected_environment_approval_rules USING btree (required_approvals, created_at);
|
||||
|
||||
CREATE INDEX index_personal_access_tokens_on_id_and_created_at ON personal_access_tokens USING btree (id, created_at);
|
||||
|
||||
CREATE UNIQUE INDEX index_personal_access_tokens_on_token_digest ON personal_access_tokens USING btree (token_digest);
|
||||
|
||||
CREATE INDEX index_personal_access_tokens_on_user_id ON personal_access_tokens USING btree (user_id);
|
||||
|
|
|
@ -1048,7 +1048,8 @@ The [secure files API](../api/secure_files.md) enforces the following limits:
|
|||
|
||||
## Changelog API limits
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default.
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default.
|
||||
> - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/33893) in GitLab 15.3.
|
||||
|
||||
The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits:
|
||||
|
||||
|
|
|
@ -384,7 +384,7 @@ POST /projects/:id/releases
|
|||
| `assets:links` | array of hash | no | An array of assets links. |
|
||||
| `assets:links:name`| string | required by: `assets:links` | The name of the link. Link names must be unique within the release. |
|
||||
| `assets:links:url` | string | required by: `assets:links` | The URL of the link. Link URLs must be unique within the release. |
|
||||
| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets).
|
||||
| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets).
|
||||
| `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`.
|
||||
| `released_at` | datetime | no | The date when the release is/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). |
|
||||
|
||||
|
|
|
@ -93,14 +93,14 @@ Create an asset as a link from a Release.
|
|||
POST /projects/:id/releases/:tag_name/assets/links
|
||||
```
|
||||
|
||||
| Attribute | Type | Required | Description |
|
||||
| ------------- | -------------- | -------- | ---------------------------------------------------------------------------------------------------------------- |
|
||||
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). |
|
||||
| `tag_name` | string | yes | The tag associated with the Release. |
|
||||
| `name` | string | yes | The name of the link. Link names must be unique within the release. |
|
||||
| `url` | string | yes | The URL of the link. Link URLs must be unique within the release. |
|
||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). |
|
||||
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
||||
| Attribute | Type | Required | Description |
|
||||
|-------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------------|
|
||||
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). |
|
||||
| `tag_name` | string | yes | The tag associated with the Release. |
|
||||
| `name` | string | yes | The name of the link. Link names must be unique in the release. |
|
||||
| `url` | string | yes | The URL of the link. Link URLs must be unique in the release. |
|
||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). |
|
||||
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
||||
|
||||
Example request:
|
||||
|
||||
|
@ -141,7 +141,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id
|
|||
| `link_id` | integer | yes | The ID of the link. |
|
||||
| `name` | string | no | The name of the link. |
|
||||
| `url` | string | no | The URL of the link. |
|
||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets).
|
||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets).
|
||||
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
||||
|
||||
NOTE:
|
||||
|
|
|
@ -136,7 +136,7 @@ As such, the target size of a physical table after refactoring depends on the si
|
|||
There is no standard solution to reduce table sizes - there are many!
|
||||
|
||||
1. **Retention**: Delete unnecessary data, for example expire old and unneeded records.
|
||||
1. **Remove STI**: We still use [single-table inheritance](../../../development/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables.
|
||||
1. **Remove STI**: We still use [single-table inheritance](../../../development/database/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables.
|
||||
1. **Index optimization**: Drop unnecessary indexes and consolidate overlapping indexes if possible.
|
||||
1. **Optimise data types**: Review data type decisions and optimise data types where possible (example: use integer instead of text for an enum column)
|
||||
1. **Partitioning**: Apply a partitioning scheme if there is a common access dimension.
|
||||
|
|
|
@ -2991,7 +2991,7 @@ released_at: '2021-03-15T08:00:00Z'
|
|||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271454) in GitLab 13.12.
|
||||
|
||||
Use `release:assets:links` to include [asset links](../../user/project/releases/index.md#release-assets) in the release.
|
||||
Use `release:assets:links` to include [asset links](../../user/project/releases/release_fields.md#release-assets) in the release.
|
||||
|
||||
Requires `release-cli` version v0.4.0 or later.
|
||||
|
||||
|
|
|
@ -1,410 +1,11 @@
|
|||
---
|
||||
stage: Data Stores
|
||||
group: Database
|
||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
redirect_to: 'database/adding_database_indexes.md'
|
||||
remove_date: '2022-11-05'
|
||||
---
|
||||
|
||||
# Adding Database Indexes
|
||||
This document was moved to [another location](database/adding_database_indexes.md).
|
||||
|
||||
Indexes can be used to speed up database queries, but when should you add a new
|
||||
index? Traditionally the answer to this question has been to add an index for
|
||||
every column used for filtering or joining data. For example, consider the
|
||||
following query:
|
||||
|
||||
```sql
|
||||
SELECT *
|
||||
FROM projects
|
||||
WHERE user_id = 2;
|
||||
```
|
||||
|
||||
Here we are filtering by the `user_id` column and as such a developer may decide
|
||||
to index this column.
|
||||
|
||||
While in certain cases indexing columns using the above approach may make sense,
|
||||
it can actually have a negative impact. Whenever you write data to a table, any
|
||||
existing indexes must also be updated. The more indexes there are, the slower this
|
||||
can potentially become. Indexes can also take up significant disk space, depending
|
||||
on the amount of data indexed and the index type. For example, PostgreSQL offers
|
||||
`GIN` indexes which can be used to index certain data types that cannot be
|
||||
indexed by regular B-tree indexes. These indexes, however, generally take up more
|
||||
data and are slower to update compared to B-tree indexes.
|
||||
|
||||
Because of all this, it's important make the following considerations
|
||||
when adding a new index:
|
||||
|
||||
1. Do the new queries re-use as many existing indexes as possible?
|
||||
1. Is there enough data that using an index is faster than iterating over
|
||||
rows in the table?
|
||||
1. Is the overhead of maintaining the index worth the reduction in query
|
||||
timings?
|
||||
|
||||
## Re-using Queries
|
||||
|
||||
The first step is to make sure your query re-uses as many existing indexes as
|
||||
possible. For example, consider the following query:
|
||||
|
||||
```sql
|
||||
SELECT *
|
||||
FROM todos
|
||||
WHERE user_id = 123
|
||||
AND state = 'open';
|
||||
```
|
||||
|
||||
Now imagine we already have an index on the `user_id` column but not on the
|
||||
`state` column. One may think this query performs badly due to `state` being
|
||||
unindexed. In reality the query may perform just fine given the index on
|
||||
`user_id` can filter out enough rows.
|
||||
|
||||
The best way to determine if indexes are re-used is to run your query using
|
||||
`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering,
|
||||
you may find an extra index doesn't make much, if any, difference.
|
||||
|
||||
In short:
|
||||
|
||||
1. Try to write your query in such a way that it re-uses as many existing
|
||||
indexes as possible.
|
||||
1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most
|
||||
ideal query.
|
||||
|
||||
## Data Size
|
||||
|
||||
A database may not use an index even when a regular sequence scan
|
||||
(iterating over all rows) is faster, especially for small tables.
|
||||
|
||||
Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows.
|
||||
You may _not_ want to add an index if the table size is small (<`1,000` records),
|
||||
or if existing indexes already filter out enough rows.
|
||||
|
||||
## Maintenance Overhead
|
||||
|
||||
Indexes have to be updated on every table write. In the case of PostgreSQL, _all_
|
||||
existing indexes are updated whenever data is written to a table. As a
|
||||
result, having many indexes on the same table slows down writes. It's therefore important
|
||||
to balance query performance with the overhead of maintaining an extra index.
|
||||
|
||||
Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases
|
||||
INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth
|
||||
it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE
|
||||
timings are unaffected.
|
||||
|
||||
## Finding Unused Indexes
|
||||
|
||||
To see which indexes are unused you can run the following query:
|
||||
|
||||
```sql
|
||||
SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass))
|
||||
FROM pg_stat_all_indexes
|
||||
WHERE schemaname = 'public'
|
||||
AND idx_scan = 0
|
||||
AND idx_tup_read = 0
|
||||
AND idx_tup_fetch = 0
|
||||
ORDER BY pg_relation_size(indexrelname::regclass) desc;
|
||||
```
|
||||
|
||||
This query outputs a list containing all indexes that are never used and sorts
|
||||
them by indexes sizes in descending order. This query helps in
|
||||
determining whether existing indexes are still required. More information on
|
||||
the meaning of the various columns can be found at
|
||||
<https://www.postgresql.org/docs/current/monitoring-stats.html>.
|
||||
|
||||
To determine if an index is still being used on production, use the following
|
||||
Thanos query with your index name:
|
||||
|
||||
```sql
|
||||
sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m]))
|
||||
```
|
||||
|
||||
Because the query output relies on the actual usage of your database, it
|
||||
may be affected by factors such as:
|
||||
|
||||
- Certain queries never being executed, thus not being able to use certain
|
||||
indexes.
|
||||
- Certain tables having little data, resulting in PostgreSQL using sequence
|
||||
scans instead of index scans.
|
||||
|
||||
This data is only reliable for a frequently used database with
|
||||
plenty of data, and using as many GitLab features as possible.
|
||||
|
||||
## Requirements for naming indexes
|
||||
|
||||
Indexes with complex definitions must be explicitly named rather than
|
||||
relying on the implicit naming behavior of migration methods. In short,
|
||||
that means you **must** provide an explicit name argument for an index
|
||||
created with one or more of the following options:
|
||||
|
||||
- `where`
|
||||
- `using`
|
||||
- `order`
|
||||
- `length`
|
||||
- `type`
|
||||
- `opclass`
|
||||
|
||||
### Considerations for index names
|
||||
|
||||
Check our [Constraints naming conventions](database/constraint_naming_convention.md) page.
|
||||
|
||||
### Why explicit names are required
|
||||
|
||||
As Rails is database agnostic, it generates an index name only
|
||||
from the required options of all indexes: table name and column names.
|
||||
For example, imagine the following two indexes are created in a migration:
|
||||
|
||||
```ruby
|
||||
def up
|
||||
add_index :my_table, :my_column
|
||||
|
||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||
end
|
||||
```
|
||||
|
||||
Creation of the second index would fail, because Rails would generate
|
||||
the same name for both indexes.
|
||||
|
||||
This naming issue is further complicated by the behavior of the `index_exists?` method.
|
||||
It considers only the table name, column names, and uniqueness specification
|
||||
of the index when making a comparison. Consider:
|
||||
|
||||
```ruby
|
||||
def up
|
||||
unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL')
|
||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
The call to `index_exists?` returns true if **any** index exists on
|
||||
`:my_table` and `:my_column`, and index creation is bypassed.
|
||||
|
||||
The `add_concurrent_index` helper is a requirement for creating indexes
|
||||
on populated tables. Because it cannot be used inside a transactional
|
||||
migration, it has a built-in check that detects if the index already
|
||||
exists. In the event a match is found, index creation is skipped.
|
||||
Without an explicit name argument, Rails can return a false positive
|
||||
for `index_exists?`, causing a required index to not be created
|
||||
properly. By always requiring a name for certain types of indexes, the
|
||||
chance of error is greatly reduced.
|
||||
|
||||
## Temporary indexes
|
||||
|
||||
There may be times when an index is only needed temporarily.
|
||||
|
||||
For example, in a migration, a column of a table might be conditionally
|
||||
updated. To query which columns must be updated in the
|
||||
[query performance guidelines](query_performance.md), an index is needed
|
||||
that would otherwise not be used.
|
||||
|
||||
In these cases, consider a temporary index. To specify a
|
||||
temporary index:
|
||||
|
||||
1. Prefix the index name with `tmp_` and follow the [naming conventions](database/constraint_naming_convention.md).
|
||||
1. Create a follow-up issue to remove the index in the next (or future) milestone.
|
||||
1. Add a comment in the migration mentioning the removal issue.
|
||||
|
||||
A temporary migration would look like:
|
||||
|
||||
```ruby
|
||||
INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled'
|
||||
|
||||
def up
|
||||
# Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234
|
||||
add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
remove_concurrent_index_by_name :projects, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
## Create indexes asynchronously
|
||||
|
||||
For very large tables, index creation can be a challenge to manage.
|
||||
While `add_concurrent_index` creates indexes in a way that does not block
|
||||
normal traffic, it can still be problematic when index creation runs for
|
||||
many hours. Necessary database operations like `autovacuum` cannot run, and
|
||||
on GitLab.com, the deployment process is blocked waiting for index
|
||||
creation to finish.
|
||||
|
||||
To limit impact on GitLab.com, a process exists to create indexes
|
||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||
index creation can proceed at a lower level of risk.
|
||||
|
||||
### Schedule index creation for a low-impact time
|
||||
|
||||
1. [Schedule the index to be created](#schedule-the-index-to-be-created).
|
||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||
|
||||
### Schedule the index to be created
|
||||
|
||||
Create an MR with a post-deployment migration which prepares the index
|
||||
for asynchronous creation. An example of creating an index using
|
||||
the asynchronous index helpers can be seen in the block below. This migration
|
||||
enters the index name and definition into the `postgres_async_indexes`
|
||||
table. The process that runs on weekends pulls indexes from this
|
||||
table and attempt to create them.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
def up
|
||||
prepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
### Verify the MR was deployed and the index exists in production
|
||||
|
||||
You can verify if the post-deploy migration was executed on GitLab.com by:
|
||||
|
||||
- Executing `/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
|
||||
the post-deploy migration has been executed in the production database. More details in this
|
||||
[guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
|
||||
- Use a meta-command in #database-lab, such as: `\d <index_name>`.
|
||||
- Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||
- Ask someone in #database to check if the index exists.
|
||||
- With proper access, you can also verify directly on production or in a
|
||||
production clone.
|
||||
|
||||
### Add a migration to create the index synchronously
|
||||
|
||||
After the index is verified to exist on the production database, create a second
|
||||
merge request that adds the index synchronously. The schema changes must be
|
||||
updated and committed to `structure.sql` in this second merge request.
|
||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||
migration as expected for other installations. The below block
|
||||
demonstrates how to create the second migration for the previous
|
||||
asynchronous example.
|
||||
|
||||
**WARNING:**
|
||||
Verify that the index exists in production before merging a second migration with `add_concurrent_index`.
|
||||
If the second migration is deployed before the index has been created,
|
||||
the index is created synchronously when the second migration executes.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
## Test database index changes locally
|
||||
|
||||
You must test the database index changes locally before creating a merge request.
|
||||
|
||||
### Verify indexes created asynchronously
|
||||
|
||||
Use the asynchronous index helpers on your local environment to test changes for creating an index:
|
||||
|
||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||
1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table.
|
||||
1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously.
|
||||
1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists.
|
||||
|
||||
## Drop indexes asynchronously
|
||||
|
||||
For very large tables, index destruction can be a challenge to manage.
|
||||
While `remove_concurrent_index` removes indexes in a way that does not block
|
||||
normal traffic, it can still be problematic if index destruction runs for
|
||||
during autovacuum. Necessary database operations like `autovacuum` cannot run, and
|
||||
the deployment process on GitLab.com is blocked while waiting for index
|
||||
destruction to finish.
|
||||
|
||||
To limit the impact on GitLab.com, use the following process to remove indexes
|
||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||
index destruction can proceed at a lower level of risk.
|
||||
|
||||
1. [Schedule the index to be removed](#schedule-the-index-to-be-removed).
|
||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||
|
||||
### Schedule the index to be removed
|
||||
|
||||
Create an MR with a post-deployment migration which prepares the index
|
||||
for asynchronous destruction. For example. to destroy an index using
|
||||
the asynchronous index helpers:
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
def up
|
||||
prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
This migration enters the index name and definition into the `postgres_async_indexes`
|
||||
table. The process that runs on weekends pulls indexes from this table and attempt
|
||||
to remove them.
|
||||
|
||||
You must test the database index changes locally before creating a merge request.
|
||||
|
||||
### Verify the MR was deployed and the index exists in production
|
||||
|
||||
You can verify if the MR was deployed to GitLab.com with
|
||||
`/chatops run auto_deploy status <merge_sha>`. To verify the existence of
|
||||
the index, you can:
|
||||
|
||||
- Use a meta-command in `#database-lab`, for example: `\d <index_name>`.
|
||||
- Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||
- Ask someone in `#database` to check if the index exists.
|
||||
- If you have access, you can verify directly on production or in a
|
||||
production clone.
|
||||
|
||||
### Add a migration to destroy the index synchronously
|
||||
|
||||
After you verify the index exists in the production database, create a second
|
||||
merge request that removes the index synchronously. The schema changes must be
|
||||
updated and committed to `structure.sql` in this second merge request.
|
||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||
migration as expected for other installations. For example, to
|
||||
create the second migration for the previous asynchronous example:
|
||||
|
||||
**WARNING:**
|
||||
Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`.
|
||||
If the second migration is deployed before the index has been destroyed,
|
||||
the index is destroyed synchronously when the second migration executes.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
add_concurrent_index :ci_builds, :some_column, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
### Verify indexes removed asynchronously
|
||||
|
||||
To test changes for removing an index, use the asynchronous index helpers on your local environment:
|
||||
|
||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||
1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table.
|
||||
1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously.
|
||||
1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
|
||||
command `gdk psql` and run `\d <index_name>` to check that the destroyed index no longer exists.
|
||||
<!-- This redirect file can be deleted after <2022-11-05>. -->
|
||||
<!-- Redirects that point to other docs in the same project expire in three months. -->
|
||||
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
|
||||
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
|
||||
|
|
|
@ -0,0 +1,410 @@
|
|||
---
|
||||
stage: Data Stores
|
||||
group: Database
|
||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
---
|
||||
|
||||
# Adding Database Indexes
|
||||
|
||||
Indexes can be used to speed up database queries, but when should you add a new
|
||||
index? Traditionally the answer to this question has been to add an index for
|
||||
every column used for filtering or joining data. For example, consider the
|
||||
following query:
|
||||
|
||||
```sql
|
||||
SELECT *
|
||||
FROM projects
|
||||
WHERE user_id = 2;
|
||||
```
|
||||
|
||||
Here we are filtering by the `user_id` column and as such a developer may decide
|
||||
to index this column.
|
||||
|
||||
While in certain cases indexing columns using the above approach may make sense,
|
||||
it can actually have a negative impact. Whenever you write data to a table, any
|
||||
existing indexes must also be updated. The more indexes there are, the slower this
|
||||
can potentially become. Indexes can also take up significant disk space, depending
|
||||
on the amount of data indexed and the index type. For example, PostgreSQL offers
|
||||
`GIN` indexes which can be used to index certain data types that cannot be
|
||||
indexed by regular B-tree indexes. These indexes, however, generally take up more
|
||||
data and are slower to update compared to B-tree indexes.
|
||||
|
||||
Because of all this, it's important make the following considerations
|
||||
when adding a new index:
|
||||
|
||||
1. Do the new queries re-use as many existing indexes as possible?
|
||||
1. Is there enough data that using an index is faster than iterating over
|
||||
rows in the table?
|
||||
1. Is the overhead of maintaining the index worth the reduction in query
|
||||
timings?
|
||||
|
||||
## Re-using Queries
|
||||
|
||||
The first step is to make sure your query re-uses as many existing indexes as
|
||||
possible. For example, consider the following query:
|
||||
|
||||
```sql
|
||||
SELECT *
|
||||
FROM todos
|
||||
WHERE user_id = 123
|
||||
AND state = 'open';
|
||||
```
|
||||
|
||||
Now imagine we already have an index on the `user_id` column but not on the
|
||||
`state` column. One may think this query performs badly due to `state` being
|
||||
unindexed. In reality the query may perform just fine given the index on
|
||||
`user_id` can filter out enough rows.
|
||||
|
||||
The best way to determine if indexes are re-used is to run your query using
|
||||
`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering,
|
||||
you may find an extra index doesn't make much, if any, difference.
|
||||
|
||||
In short:
|
||||
|
||||
1. Try to write your query in such a way that it re-uses as many existing
|
||||
indexes as possible.
|
||||
1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most
|
||||
ideal query.
|
||||
|
||||
## Data Size
|
||||
|
||||
A database may not use an index even when a regular sequence scan
|
||||
(iterating over all rows) is faster, especially for small tables.
|
||||
|
||||
Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows.
|
||||
You may _not_ want to add an index if the table size is small (<`1,000` records),
|
||||
or if existing indexes already filter out enough rows.
|
||||
|
||||
## Maintenance Overhead
|
||||
|
||||
Indexes have to be updated on every table write. In the case of PostgreSQL, _all_
|
||||
existing indexes are updated whenever data is written to a table. As a
|
||||
result, having many indexes on the same table slows down writes. It's therefore important
|
||||
to balance query performance with the overhead of maintaining an extra index.
|
||||
|
||||
Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases
|
||||
INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth
|
||||
it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE
|
||||
timings are unaffected.
|
||||
|
||||
## Finding Unused Indexes
|
||||
|
||||
To see which indexes are unused you can run the following query:
|
||||
|
||||
```sql
|
||||
SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass))
|
||||
FROM pg_stat_all_indexes
|
||||
WHERE schemaname = 'public'
|
||||
AND idx_scan = 0
|
||||
AND idx_tup_read = 0
|
||||
AND idx_tup_fetch = 0
|
||||
ORDER BY pg_relation_size(indexrelname::regclass) desc;
|
||||
```
|
||||
|
||||
This query outputs a list containing all indexes that are never used and sorts
|
||||
them by indexes sizes in descending order. This query helps in
|
||||
determining whether existing indexes are still required. More information on
|
||||
the meaning of the various columns can be found at
|
||||
<https://www.postgresql.org/docs/current/monitoring-stats.html>.
|
||||
|
||||
To determine if an index is still being used on production, use the following
|
||||
Thanos query with your index name:
|
||||
|
||||
```sql
|
||||
sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m]))
|
||||
```
|
||||
|
||||
Because the query output relies on the actual usage of your database, it
|
||||
may be affected by factors such as:
|
||||
|
||||
- Certain queries never being executed, thus not being able to use certain
|
||||
indexes.
|
||||
- Certain tables having little data, resulting in PostgreSQL using sequence
|
||||
scans instead of index scans.
|
||||
|
||||
This data is only reliable for a frequently used database with
|
||||
plenty of data, and using as many GitLab features as possible.
|
||||
|
||||
## Requirements for naming indexes
|
||||
|
||||
Indexes with complex definitions must be explicitly named rather than
|
||||
relying on the implicit naming behavior of migration methods. In short,
|
||||
that means you **must** provide an explicit name argument for an index
|
||||
created with one or more of the following options:
|
||||
|
||||
- `where`
|
||||
- `using`
|
||||
- `order`
|
||||
- `length`
|
||||
- `type`
|
||||
- `opclass`
|
||||
|
||||
### Considerations for index names
|
||||
|
||||
Check our [Constraints naming conventions](constraint_naming_convention.md) page.
|
||||
|
||||
### Why explicit names are required
|
||||
|
||||
As Rails is database agnostic, it generates an index name only
|
||||
from the required options of all indexes: table name and column names.
|
||||
For example, imagine the following two indexes are created in a migration:
|
||||
|
||||
```ruby
|
||||
def up
|
||||
add_index :my_table, :my_column
|
||||
|
||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||
end
|
||||
```
|
||||
|
||||
Creation of the second index would fail, because Rails would generate
|
||||
the same name for both indexes.
|
||||
|
||||
This naming issue is further complicated by the behavior of the `index_exists?` method.
|
||||
It considers only the table name, column names, and uniqueness specification
|
||||
of the index when making a comparison. Consider:
|
||||
|
||||
```ruby
|
||||
def up
|
||||
unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL')
|
||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
The call to `index_exists?` returns true if **any** index exists on
|
||||
`:my_table` and `:my_column`, and index creation is bypassed.
|
||||
|
||||
The `add_concurrent_index` helper is a requirement for creating indexes
|
||||
on populated tables. Because it cannot be used inside a transactional
|
||||
migration, it has a built-in check that detects if the index already
|
||||
exists. In the event a match is found, index creation is skipped.
|
||||
Without an explicit name argument, Rails can return a false positive
|
||||
for `index_exists?`, causing a required index to not be created
|
||||
properly. By always requiring a name for certain types of indexes, the
|
||||
chance of error is greatly reduced.
|
||||
|
||||
## Temporary indexes
|
||||
|
||||
There may be times when an index is only needed temporarily.
|
||||
|
||||
For example, in a migration, a column of a table might be conditionally
|
||||
updated. To query which columns must be updated in the
|
||||
[query performance guidelines](../query_performance.md), an index is needed
|
||||
that would otherwise not be used.
|
||||
|
||||
In these cases, consider a temporary index. To specify a
|
||||
temporary index:
|
||||
|
||||
1. Prefix the index name with `tmp_` and follow the [naming conventions](constraint_naming_convention.md).
|
||||
1. Create a follow-up issue to remove the index in the next (or future) milestone.
|
||||
1. Add a comment in the migration mentioning the removal issue.
|
||||
|
||||
A temporary migration would look like:
|
||||
|
||||
```ruby
|
||||
INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled'
|
||||
|
||||
def up
|
||||
# Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234
|
||||
add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
remove_concurrent_index_by_name :projects, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
## Create indexes asynchronously
|
||||
|
||||
For very large tables, index creation can be a challenge to manage.
|
||||
While `add_concurrent_index` creates indexes in a way that does not block
|
||||
normal traffic, it can still be problematic when index creation runs for
|
||||
many hours. Necessary database operations like `autovacuum` cannot run, and
|
||||
on GitLab.com, the deployment process is blocked waiting for index
|
||||
creation to finish.
|
||||
|
||||
To limit impact on GitLab.com, a process exists to create indexes
|
||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||
index creation can proceed at a lower level of risk.
|
||||
|
||||
### Schedule index creation for a low-impact time
|
||||
|
||||
1. [Schedule the index to be created](#schedule-the-index-to-be-created).
|
||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||
|
||||
### Schedule the index to be created
|
||||
|
||||
Create an MR with a post-deployment migration which prepares the index
|
||||
for asynchronous creation. An example of creating an index using
|
||||
the asynchronous index helpers can be seen in the block below. This migration
|
||||
enters the index name and definition into the `postgres_async_indexes`
|
||||
table. The process that runs on weekends pulls indexes from this
|
||||
table and attempt to create them.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
def up
|
||||
prepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
### Verify the MR was deployed and the index exists in production
|
||||
|
||||
You can verify if the post-deploy migration was executed on GitLab.com by:
|
||||
|
||||
- Executing `/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
|
||||
the post-deploy migration has been executed in the production database. More details in this
|
||||
[guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
|
||||
- Use a meta-command in #database-lab, such as: `\d <index_name>`.
|
||||
- Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||
- Ask someone in #database to check if the index exists.
|
||||
- With proper access, you can also verify directly on production or in a
|
||||
production clone.
|
||||
|
||||
### Add a migration to create the index synchronously
|
||||
|
||||
After the index is verified to exist on the production database, create a second
|
||||
merge request that adds the index synchronously. The schema changes must be
|
||||
updated and committed to `structure.sql` in this second merge request.
|
||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||
migration as expected for other installations. The below block
|
||||
demonstrates how to create the second migration for the previous
|
||||
asynchronous example.
|
||||
|
||||
**WARNING:**
|
||||
Verify that the index exists in production before merging a second migration with `add_concurrent_index`.
|
||||
If the second migration is deployed before the index has been created,
|
||||
the index is created synchronously when the second migration executes.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
## Test database index changes locally
|
||||
|
||||
You must test the database index changes locally before creating a merge request.
|
||||
|
||||
### Verify indexes created asynchronously
|
||||
|
||||
Use the asynchronous index helpers on your local environment to test changes for creating an index:
|
||||
|
||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||
1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table.
|
||||
1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously.
|
||||
1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists.
|
||||
|
||||
## Drop indexes asynchronously
|
||||
|
||||
For very large tables, index destruction can be a challenge to manage.
|
||||
While `remove_concurrent_index` removes indexes in a way that does not block
|
||||
normal traffic, it can still be problematic if index destruction runs for
|
||||
during autovacuum. Necessary database operations like `autovacuum` cannot run, and
|
||||
the deployment process on GitLab.com is blocked while waiting for index
|
||||
destruction to finish.
|
||||
|
||||
To limit the impact on GitLab.com, use the following process to remove indexes
|
||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||
index destruction can proceed at a lower level of risk.
|
||||
|
||||
1. [Schedule the index to be removed](#schedule-the-index-to-be-removed).
|
||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||
|
||||
### Schedule the index to be removed
|
||||
|
||||
Create an MR with a post-deployment migration which prepares the index
|
||||
for asynchronous destruction. For example. to destroy an index using
|
||||
the asynchronous index helpers:
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
def up
|
||||
prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
This migration enters the index name and definition into the `postgres_async_indexes`
|
||||
table. The process that runs on weekends pulls indexes from this table and attempt
|
||||
to remove them.
|
||||
|
||||
You must test the database index changes locally before creating a merge request.
|
||||
|
||||
### Verify the MR was deployed and the index exists in production
|
||||
|
||||
You can verify if the MR was deployed to GitLab.com with
|
||||
`/chatops run auto_deploy status <merge_sha>`. To verify the existence of
|
||||
the index, you can:
|
||||
|
||||
- Use a meta-command in `#database-lab`, for example: `\d <index_name>`.
|
||||
- Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||
- Ask someone in `#database` to check if the index exists.
|
||||
- If you have access, you can verify directly on production or in a
|
||||
production clone.
|
||||
|
||||
### Add a migration to destroy the index synchronously
|
||||
|
||||
After you verify the index exists in the production database, create a second
|
||||
merge request that removes the index synchronously. The schema changes must be
|
||||
updated and committed to `structure.sql` in this second merge request.
|
||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||
migration as expected for other installations. For example, to
|
||||
create the second migration for the previous asynchronous example:
|
||||
|
||||
**WARNING:**
|
||||
Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`.
|
||||
If the second migration is deployed before the index has been destroyed,
|
||||
the index is destroyed synchronously when the second migration executes.
|
||||
|
||||
```ruby
|
||||
# in db/post_migrate/
|
||||
|
||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||
|
||||
disable_ddl_transaction!
|
||||
|
||||
def up
|
||||
remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME
|
||||
end
|
||||
|
||||
def down
|
||||
add_concurrent_index :ci_builds, :some_column, INDEX_NAME
|
||||
end
|
||||
```
|
||||
|
||||
### Verify indexes removed asynchronously
|
||||
|
||||
To test changes for removing an index, use the asynchronous index helpers on your local environment:
|
||||
|
||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||
1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table.
|
||||
1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously.
|
||||
1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
|
||||
command `gdk psql` and run `\d <index_name>` to check that the destroyed index no longer exists.
|
|
@ -0,0 +1,62 @@
|
|||
---
|
||||
stage: Data Stores
|
||||
group: Database
|
||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
---
|
||||
|
||||
# Database query comments with Marginalia
|
||||
|
||||
The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add
|
||||
query comments containing application related context information to PostgreSQL
|
||||
queries generated by ActiveRecord.
|
||||
|
||||
It is very useful for tracing problematic queries back to the application source.
|
||||
|
||||
An engineer during an on-call incident has the full context of a query
|
||||
and its application source from the comments.
|
||||
|
||||
## Metadata information in comments
|
||||
|
||||
Queries generated from **Rails** include the following metadata in comments:
|
||||
|
||||
- `application`
|
||||
- `correlation_id`
|
||||
- `endpoint_id`
|
||||
- `line`
|
||||
|
||||
Queries generated from **Sidekiq** workers include the following metadata
|
||||
in comments:
|
||||
|
||||
- `application`
|
||||
- `jid`
|
||||
- `correlation_id`
|
||||
- `endpoint_id`
|
||||
- `line`
|
||||
|
||||
`endpoint_id` is a single field that can represent any endpoint in the application:
|
||||
|
||||
- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`.
|
||||
- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`.
|
||||
- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`.
|
||||
|
||||
`line` is not present in production logs due to the additional overhead required.
|
||||
|
||||
Examples of queries with comments:
|
||||
|
||||
- Rails:
|
||||
|
||||
```sql
|
||||
/*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1
|
||||
```
|
||||
|
||||
- Grape:
|
||||
|
||||
```sql
|
||||
/*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1
|
||||
```
|
||||
|
||||
- Sidekiq:
|
||||
|
||||
```sql
|
||||
/*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3
|
||||
```
|
|
@ -36,17 +36,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
|||
|
||||
## Debugging
|
||||
|
||||
- Tracing the source of an SQL query using query comments with [Marginalia](../database_query_comments.md)
|
||||
- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md)
|
||||
- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs)
|
||||
|
||||
## Best practices
|
||||
|
||||
- [Adding database indexes](../adding_database_indexes.md)
|
||||
- [Adding database indexes](adding_database_indexes.md)
|
||||
- [Foreign keys & associations](../foreign_keys.md)
|
||||
- [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md)
|
||||
- [`NOT NULL` constraints](not_null_constraints.md)
|
||||
- [Strings and the Text data type](strings_and_the_text_data_type.md)
|
||||
- [Single table inheritance](../single_table_inheritance.md)
|
||||
- [Single table inheritance](single_table_inheritance.md)
|
||||
- [Polymorphic associations](polymorphic_associations.md)
|
||||
- [Serializing data](../serializing_data.md)
|
||||
- [Hash indexes](../hash_indexes.md)
|
||||
|
|
|
@ -1,62 +1,11 @@
|
|||
---
|
||||
stage: Data Stores
|
||||
group: Database
|
||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
redirect_to: 'database/database_query_comments.md'
|
||||
remove_date: '2022-11-05'
|
||||
---
|
||||
|
||||
# Database query comments with Marginalia
|
||||
This document was moved to [another location](database/database_query_comments.md).
|
||||
|
||||
The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add
|
||||
query comments containing application related context information to PostgreSQL
|
||||
queries generated by ActiveRecord.
|
||||
|
||||
It is very useful for tracing problematic queries back to the application source.
|
||||
|
||||
An engineer during an on-call incident has the full context of a query
|
||||
and its application source from the comments.
|
||||
|
||||
## Metadata information in comments
|
||||
|
||||
Queries generated from **Rails** include the following metadata in comments:
|
||||
|
||||
- `application`
|
||||
- `correlation_id`
|
||||
- `endpoint_id`
|
||||
- `line`
|
||||
|
||||
Queries generated from **Sidekiq** workers include the following metadata
|
||||
in comments:
|
||||
|
||||
- `application`
|
||||
- `jid`
|
||||
- `correlation_id`
|
||||
- `endpoint_id`
|
||||
- `line`
|
||||
|
||||
`endpoint_id` is a single field that can represent any endpoint in the application:
|
||||
|
||||
- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`.
|
||||
- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`.
|
||||
- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`.
|
||||
|
||||
`line` is not present in production logs due to the additional overhead required.
|
||||
|
||||
Examples of queries with comments:
|
||||
|
||||
- Rails:
|
||||
|
||||
```sql
|
||||
/*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1
|
||||
```
|
||||
|
||||
- Grape:
|
||||
|
||||
```sql
|
||||
/*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1
|
||||
```
|
||||
|
||||
- Sidekiq:
|
||||
|
||||
```sql
|
||||
/*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3
|
||||
```
|
||||
<!-- This redirect file can be deleted after <2022-11-05>. -->
|
||||
<!-- Redirects that point to other docs in the same project expire in three months. -->
|
||||
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
|
||||
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
|
||||
|
|
|
@ -120,7 +120,7 @@ columns manually for existing tables as this causes confusion to
|
|||
other people using `db/structure.sql` generated by Rails.
|
||||
|
||||
NOTE:
|
||||
[Creating an index asynchronously requires two merge requests.](adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously)
|
||||
[Creating an index asynchronously requires two merge requests.](database/adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously)
|
||||
When done, commit the schema change in the merge request
|
||||
that adds the index with `add_concurrent_index`.
|
||||
|
||||
|
@ -612,7 +612,7 @@ might not be required, like:
|
|||
|
||||
Additionally, wide indexes are not required to match all filter criteria of queries, we just need
|
||||
to cover enough columns so that the index lookup has a small enough selectivity. Please review our
|
||||
[Adding Database indexes](adding_database_indexes.md) guide for more details.
|
||||
[Adding Database indexes](database/adding_database_indexes.md) guide for more details.
|
||||
|
||||
When adding an index to a non-empty table make sure to use the method
|
||||
`add_concurrent_index` instead of the regular `add_index` method.
|
||||
|
@ -641,7 +641,7 @@ end
|
|||
|
||||
You must explicitly name indexes that are created with more complex
|
||||
definitions beyond table name, column names, and uniqueness constraint.
|
||||
Consult the [Adding Database Indexes](adding_database_indexes.md#requirements-for-naming-indexes)
|
||||
Consult the [Adding Database Indexes](database/adding_database_indexes.md#requirements-for-naming-indexes)
|
||||
guide for more details.
|
||||
|
||||
If you need to add a unique index, please keep in mind there is the possibility
|
||||
|
@ -659,7 +659,7 @@ If a migration requires conditional logic based on the absence or
|
|||
presence of an index, you must test for existence of that index using
|
||||
its name. This helps avoids problems with how Rails compares index definitions,
|
||||
which can lead to unexpected results. For more details, review the
|
||||
[Adding Database Indexes](adding_database_indexes.md#why-explicit-names-are-required)
|
||||
[Adding Database Indexes](database/adding_database_indexes.md#why-explicit-names-are-required)
|
||||
guide.
|
||||
|
||||
The easiest way to test for existence of an index by name is to use the
|
||||
|
@ -1198,7 +1198,7 @@ If using a model in the migrations, you should first
|
|||
using `reset_column_information`.
|
||||
|
||||
If using a model that leverages single table inheritance (STI), there are [special
|
||||
considerations](single_table_inheritance.md#in-migrations).
|
||||
considerations](database/single_table_inheritance.md#in-migrations).
|
||||
|
||||
This avoids problems where a column that you are using was altered and cached
|
||||
in a previous migration.
|
||||
|
|
|
@ -32,12 +32,10 @@ When you create a release, or after, you can:
|
|||
- Add release notes.
|
||||
- Add a message for the Git tag associated with the release.
|
||||
- [Associate milestones with it](#associate-milestones-with-a-release).
|
||||
- Attach [release assets](#release-assets), like runbooks or packages.
|
||||
- Attach [release assets](release_fields.md#release-assets), like runbooks or packages.
|
||||
|
||||
## View releases
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8.
|
||||
|
||||
To view a list of releases:
|
||||
|
||||
- On the left sidebar, select **Deployments > Releases**, or
|
||||
|
@ -81,18 +79,18 @@ To create a release in the Releases page:
|
|||
|
||||
1. On the top bar, select **Menu > Projects** and find your project.
|
||||
1. On the left sidebar, select **Deployments > Releases** and select **New release**.
|
||||
1. From the [**Tag name**](#tag-name) dropdown, either:
|
||||
1. From the [**Tag name**](release_fields.md#tag-name) dropdown, either:
|
||||
- Select an existing Git tag. Selecting an existing tag that is already associated with a release
|
||||
results in a validation error.
|
||||
- Enter a new Git tag name.
|
||||
1. From the **Create from** dropdown, select a branch or commit SHA to use when creating the
|
||||
new tag.
|
||||
1. Optional. Enter additional information about the release, including:
|
||||
- [Title](#title).
|
||||
- [Title](release_fields.md#title).
|
||||
- [Milestones](#associate-milestones-with-a-release).
|
||||
- [Release notes](#release-notes-description).
|
||||
- [Release notes](release_fields.md#release-notes-description).
|
||||
- Whether or not to include the [Tag message](../../../topics/git/tags.md).
|
||||
- [Asset links](#links).
|
||||
- [Asset links](release_fields.md#links).
|
||||
1. Select **Create release**.
|
||||
|
||||
### Create a release in the Tags page
|
||||
|
@ -298,9 +296,6 @@ is not available.
|
|||
|
||||
## Edit a release
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6.
|
||||
> - Asset link editing [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10.
|
||||
|
||||
Only users with at least the Developer role can edit releases.
|
||||
Read more about [Release permissions](#release-permissions).
|
||||
|
||||
|
@ -428,277 +423,6 @@ complete overlapping period.
|
|||
|
||||
For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md).
|
||||
|
||||
## Release fields
|
||||
|
||||
The following fields are available when you create or edit a release.
|
||||
|
||||
### Title
|
||||
|
||||
The release title can be customized using the **Release title** field when
|
||||
creating or editing a release. If no title is provided, the release's tag name
|
||||
is used instead.
|
||||
|
||||
### Tag name
|
||||
|
||||
The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/)
|
||||
for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the
|
||||
[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning).
|
||||
|
||||
For example, for GitLab version `10.5.7`:
|
||||
|
||||
- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`.
|
||||
- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`.
|
||||
- `7` represents the patch number.
|
||||
|
||||
Any part of the version number can be multiple digits, for example, `13.10.11`.
|
||||
|
||||
### Release notes description
|
||||
|
||||
Every release has a description. You can add any text you like, but we recommend
|
||||
including a changelog to describe the content of your release. This helps users
|
||||
quickly scan the differences between each release you publish.
|
||||
|
||||
[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can
|
||||
be included in Release note descriptions by selecting **Include tag message in
|
||||
the release notes**.
|
||||
|
||||
Description supports [Markdown](../../markdown.md).
|
||||
|
||||
### Release assets
|
||||
|
||||
A release contains the following types of assets:
|
||||
|
||||
- [Source code](#source-code)
|
||||
- [Link](#links)
|
||||
|
||||
#### Source code
|
||||
|
||||
GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar`
|
||||
archived source code from the given Git tag. These are read-only assets.
|
||||
|
||||
#### Links
|
||||
|
||||
A link is any URL which can point to whatever you like: documentation, built
|
||||
binaries, or other related materials. These can be both internal or external
|
||||
links from your GitLab instance.
|
||||
Each link as an asset has the following attributes:
|
||||
|
||||
| Attribute | Description | Required |
|
||||
| ---- | ----------- | --- |
|
||||
| `name` | The name of the link. | Yes |
|
||||
| `url` | The URL to download a file. | Yes |
|
||||
| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No |
|
||||
| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No |
|
||||
|
||||
##### Permanent link to latest release
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||
|
||||
Latest release page is accessible through a permanent URL.
|
||||
GitLab will redirect to the latest release page URL when it is visited.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/permalink/latest
|
||||
```
|
||||
|
||||
We also support, suffix path carry forward on the redirect to the latest release.
|
||||
Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`.
|
||||
|
||||
Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage.
|
||||
|
||||
###### Sorting preferences
|
||||
|
||||
By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945).
|
||||
|
||||
##### Permanent links to release assets
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9.
|
||||
|
||||
The assets associated with a release are accessible through a permanent URL.
|
||||
GitLab always redirects this URL to the actual asset
|
||||
location, so even if the assets move to a different location, you can continue
|
||||
to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/:release/downloads/:filepath
|
||||
```
|
||||
|
||||
If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org`
|
||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "linux amd64",
|
||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||
"link_type": "other"
|
||||
}
|
||||
```
|
||||
|
||||
This asset has a direct link of:
|
||||
|
||||
```plaintext
|
||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64
|
||||
```
|
||||
|
||||
The physical location of the asset can change at any time and the direct link remains unchanged.
|
||||
|
||||
##### Permanent links to latest release assets
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||
|
||||
The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath
|
||||
```
|
||||
|
||||
If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org`
|
||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "linux amd64",
|
||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||
"link_type": "other"
|
||||
}
|
||||
```
|
||||
|
||||
This asset has a direct link of:
|
||||
|
||||
```plaintext
|
||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64
|
||||
```
|
||||
|
||||
##### Link Types
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1.
|
||||
|
||||
The four types of links are "Runbook," "Package," "Image," and "Other."
|
||||
The `link_type` parameter accepts one of the following four values:
|
||||
|
||||
- `runbook`
|
||||
- `package`
|
||||
- `image`
|
||||
- `other` (default)
|
||||
|
||||
This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
|
||||
|
||||
##### Use a generic package for attaching binaries
|
||||
|
||||
You can use [generic packages](../../packages/generic_packages/index.md)
|
||||
to store any artifacts from a release or tag pipeline,
|
||||
that can also be used for attaching binary files to an individual release entry.
|
||||
You basically need to:
|
||||
|
||||
1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file).
|
||||
1. [Attach the package link to the release](#links).
|
||||
|
||||
The following example generates release assets, publishes them
|
||||
as a generic package, and then creates a release:
|
||||
|
||||
```yaml
|
||||
stages:
|
||||
- build
|
||||
- upload
|
||||
- release
|
||||
|
||||
variables:
|
||||
# Package version can only contain numbers (0-9), and dots (.).
|
||||
# Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion.
|
||||
# See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file
|
||||
PACKAGE_VERSION: "1.2.3"
|
||||
DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}"
|
||||
LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}"
|
||||
PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}"
|
||||
|
||||
build:
|
||||
stage: build
|
||||
image: alpine:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- mkdir bin
|
||||
- echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY}
|
||||
- echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY}
|
||||
artifacts:
|
||||
paths:
|
||||
- bin/
|
||||
|
||||
upload:
|
||||
stage: upload
|
||||
image: curlimages/curl:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- |
|
||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}"
|
||||
- |
|
||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}"
|
||||
|
||||
release:
|
||||
# Caution, as of 2021-02-02 these assets links require a login, see:
|
||||
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
|
||||
stage: release
|
||||
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- |
|
||||
release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
|
||||
--assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \
|
||||
--assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}"
|
||||
```
|
||||
|
||||
PowerShell users may need to escape the double quote `"` inside a JSON
|
||||
string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json`
|
||||
before passing on to the `release-cli`.
|
||||
For example:
|
||||
|
||||
```yaml
|
||||
release:
|
||||
script:
|
||||
- $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}"
|
||||
- $env:assetjson = $env:asset | ConvertTo-Json
|
||||
- release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson
|
||||
```
|
||||
|
||||
NOTE:
|
||||
Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md)
|
||||
links to a release is not recommended, because artifacts are ephemeral and
|
||||
are used to pass data in the same pipeline. This means there's a risk that
|
||||
they could either expire or someone might manually delete them.
|
||||
|
||||
#### Number of new and total features **(FREE SAAS)**
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5.
|
||||
|
||||
On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project.
|
||||
|
||||
![Feature count](img/feature_count_v14_6.png "Number of features in a release")
|
||||
|
||||
The totals are displayed on [shields](https://shields.io/) and are generated per release by
|
||||
[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake).
|
||||
|
||||
| Item | Formula |
|
||||
| ------ | ------ |
|
||||
| `New features` | Total count of release posts across all tiers for a single release in the project. |
|
||||
| `Total features` | Total count of release posts in reverse order for all releases in the project. |
|
||||
|
||||
The counts are also shown by license tier.
|
||||
|
||||
| Item | Formula |
|
||||
| ------ | ------ |
|
||||
| `New features` | Total count of release posts across a single tier for a single release in the project. |
|
||||
| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. |
|
||||
|
||||
## Release evidence
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6.
|
||||
|
@ -847,7 +571,7 @@ In the API:
|
|||
- Users with the Guest role
|
||||
have read and download access to the project releases.
|
||||
This includes associated Git-tag-names, release description, author information of the releases.
|
||||
However, other repository-related information, such as [source code](#source-code), [release evidence](#release-evidence) are redacted.
|
||||
However, other repository-related information, such as [source code](release_fields.md#source-code), [release evidence](#release-evidence) are redacted.
|
||||
|
||||
### Create, update, and delete a release and its assets
|
||||
|
||||
|
|
|
@ -0,0 +1,274 @@
|
|||
---
|
||||
stage: Release
|
||||
group: Release
|
||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
---
|
||||
|
||||
# Release fields
|
||||
|
||||
The following fields are available when you create or edit a release.
|
||||
|
||||
## Title
|
||||
|
||||
The release title can be customized using the **Release title** field when
|
||||
creating or editing a release. If no title is provided, the release's tag name
|
||||
is used instead.
|
||||
|
||||
## Tag name
|
||||
|
||||
The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/)
|
||||
for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the
|
||||
[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning).
|
||||
|
||||
For example, for GitLab version `10.5.7`:
|
||||
|
||||
- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`.
|
||||
- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`.
|
||||
- `7` represents the patch number.
|
||||
|
||||
Any part of the version number can be multiple digits, for example, `13.10.11`.
|
||||
|
||||
## Release notes description
|
||||
|
||||
Every release has a description. You can add any text you like, but we recommend
|
||||
including a changelog to describe the content of your release. This helps users
|
||||
quickly scan the differences between each release you publish.
|
||||
|
||||
[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can
|
||||
be included in Release note descriptions by selecting **Include tag message in
|
||||
the release notes**.
|
||||
|
||||
Description supports [Markdown](../../markdown.md).
|
||||
|
||||
## Release assets
|
||||
|
||||
A release contains the following types of assets:
|
||||
|
||||
- [Source code](#source-code)
|
||||
- [Link](#links)
|
||||
|
||||
### Source code
|
||||
|
||||
GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar`
|
||||
archived source code from the given Git tag. These are read-only assets.
|
||||
|
||||
### Links
|
||||
|
||||
A link is any URL which can point to whatever you like: documentation, built
|
||||
binaries, or other related materials. These can be both internal or external
|
||||
links from your GitLab instance.
|
||||
Each link as an asset has the following attributes:
|
||||
|
||||
| Attribute | Description | Required |
|
||||
|-------------|--------------------------------------------------------------------------------------------------------------|----------|
|
||||
| `name` | The name of the link. | Yes |
|
||||
| `url` | The URL to download a file. | Yes |
|
||||
| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No |
|
||||
| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No |
|
||||
|
||||
#### Permanent link to latest release
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||
|
||||
Latest release page is accessible through a permanent URL.
|
||||
GitLab redirects to the latest release page URL when it is visited.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/permalink/latest
|
||||
```
|
||||
|
||||
We also support, suffix path carry forward on the redirect to the latest release.
|
||||
Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`.
|
||||
|
||||
Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage.
|
||||
|
||||
##### Sorting preferences
|
||||
|
||||
By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945).
|
||||
|
||||
#### Permanent links to release assets
|
||||
|
||||
The assets associated with a release are accessible through a permanent URL.
|
||||
GitLab always redirects this URL to the actual asset
|
||||
location, so even if the assets move to a different location, you can continue
|
||||
to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/:release/downloads/:filepath
|
||||
```
|
||||
|
||||
If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org`
|
||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "linux amd64",
|
||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||
"link_type": "other"
|
||||
}
|
||||
```
|
||||
|
||||
This asset has a direct link of:
|
||||
|
||||
```plaintext
|
||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64
|
||||
```
|
||||
|
||||
The physical location of the asset can change at any time and the direct link remains unchanged.
|
||||
|
||||
#### Permanent links to latest release assets
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||
|
||||
The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*.
|
||||
|
||||
The format of the URL is:
|
||||
|
||||
```plaintext
|
||||
https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath
|
||||
```
|
||||
|
||||
If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org`
|
||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "linux amd64",
|
||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||
"link_type": "other"
|
||||
}
|
||||
```
|
||||
|
||||
This asset has a direct link of:
|
||||
|
||||
```plaintext
|
||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64
|
||||
```
|
||||
|
||||
#### Link Types
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1.
|
||||
|
||||
The four types of links are "Runbook," "Package," "Image," and "Other."
|
||||
The `link_type` parameter accepts one of the following four values:
|
||||
|
||||
- `runbook`
|
||||
- `package`
|
||||
- `image`
|
||||
- `other` (default)
|
||||
|
||||
This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
|
||||
|
||||
#### Use a generic package for attaching binaries
|
||||
|
||||
You can use [generic packages](../../packages/generic_packages/index.md)
|
||||
to store any artifacts from a release or tag pipeline,
|
||||
that can also be used for attaching binary files to an individual release entry.
|
||||
You basically need to:
|
||||
|
||||
1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file).
|
||||
1. [Attach the package link to the release](#links).
|
||||
|
||||
The following example generates release assets, publishes them
|
||||
as a generic package, and then creates a release:
|
||||
|
||||
```yaml
|
||||
stages:
|
||||
- build
|
||||
- upload
|
||||
- release
|
||||
|
||||
variables:
|
||||
# Package version can only contain numbers (0-9), and dots (.).
|
||||
# Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion.
|
||||
# See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file
|
||||
PACKAGE_VERSION: "1.2.3"
|
||||
DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}"
|
||||
LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}"
|
||||
PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}"
|
||||
|
||||
build:
|
||||
stage: build
|
||||
image: alpine:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- mkdir bin
|
||||
- echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY}
|
||||
- echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY}
|
||||
artifacts:
|
||||
paths:
|
||||
- bin/
|
||||
|
||||
upload:
|
||||
stage: upload
|
||||
image: curlimages/curl:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- |
|
||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}"
|
||||
- |
|
||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}"
|
||||
|
||||
release:
|
||||
# Caution, as of 2021-02-02 these assets links require a login, see:
|
||||
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
|
||||
stage: release
|
||||
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
||||
rules:
|
||||
- if: $CI_COMMIT_TAG
|
||||
script:
|
||||
- |
|
||||
release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
|
||||
--assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \
|
||||
--assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}"
|
||||
```
|
||||
|
||||
PowerShell users may need to escape the double quote `"` inside a JSON
|
||||
string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json`
|
||||
before passing on to the `release-cli`.
|
||||
For example:
|
||||
|
||||
```yaml
|
||||
release:
|
||||
script:
|
||||
- $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}"
|
||||
- $env:assetjson = $env:asset | ConvertTo-Json
|
||||
- release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson
|
||||
```
|
||||
|
||||
NOTE:
|
||||
Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md)
|
||||
links to a release is not recommended, because artifacts are ephemeral and
|
||||
are used to pass data in the same pipeline. This means there's a risk that
|
||||
they could either expire or someone might manually delete them.
|
||||
|
||||
### Number of new and total features **(FREE SAAS)**
|
||||
|
||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5.
|
||||
|
||||
On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project.
|
||||
|
||||
![Feature count](img/feature_count_v14_6.png "Number of features in a release")
|
||||
|
||||
The totals are displayed on [shields](https://shields.io/) and are generated per release by
|
||||
[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake).
|
||||
|
||||
| Item | Formula |
|
||||
|------------------|------------------------------------------------------------------------------------|
|
||||
| `New features` | Total count of release posts across all tiers for a single release in the project. |
|
||||
| `Total features` | Total count of release posts in reverse order for all releases in the project. |
|
||||
|
||||
The counts are also shown by license tier.
|
||||
|
||||
| Item | Formula |
|
||||
|------------------|-----------------------------------------------------------------------------------------------------|
|
||||
| `New features` | Total count of release posts across a single tier for a single release in the project. |
|
||||
| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. |
|
|
@ -9,7 +9,7 @@ module Gitlab
|
|||
VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze
|
||||
|
||||
def self.parse(str, parse_suffix: false)
|
||||
if str.is_a?(self.class)
|
||||
if str.is_a?(self)
|
||||
str
|
||||
elsif str && m = str.match(VERSION_REGEX)
|
||||
VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil)
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
|
||||
RSpec.describe 'Merge request > Context commits', :js do
|
||||
let(:user) { create(:user) }
|
||||
let(:project) { create(:project, :public, :repository) }
|
||||
let(:merge_request) { create(:merge_request, source_project: project) }
|
||||
|
||||
before do
|
||||
project.add_developer(user)
|
||||
|
||||
sign_in(user)
|
||||
|
||||
visit commits_project_merge_request_path(project, merge_request)
|
||||
|
||||
wait_for_requests
|
||||
end
|
||||
|
||||
it 'opens modal' do
|
||||
click_button 'Add previously merged commits'
|
||||
|
||||
expect(page).to have_selector('#add-review-item')
|
||||
expect(page).to have_content('Add or remove previously merged commits')
|
||||
end
|
||||
end
|
|
@ -3,3 +3,4 @@ export * from './to_have_tracking_attributes';
|
|||
export * from './to_match_interpolated_text';
|
||||
export * from './to_validate_json_schema';
|
||||
export * from './to_match_expected_for_markdown';
|
||||
export * from './to_equal_graphql_fixture';
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
import { isEqual } from 'lodash';
|
||||
import { stripTypenames } from 'helpers/graphql_helpers';
|
||||
|
||||
export function toEqualGraphqlFixture(received, match) {
|
||||
let clearReceived;
|
||||
let clearMatch;
|
||||
|
||||
try {
|
||||
clearReceived = JSON.parse(JSON.stringify(received));
|
||||
clearMatch = stripTypenames(match);
|
||||
} catch (e) {
|
||||
return { message: () => 'The comparator value is not an object', pass: false };
|
||||
}
|
||||
const pass = isEqual(clearReceived, clearMatch);
|
||||
// console.log(this.utils);
|
||||
const message = pass
|
||||
? () => `
|
||||
Expected to not be: ${this.utils.printExpected(clearMatch)}
|
||||
Received: ${this.utils.printReceived(clearReceived)}
|
||||
`
|
||||
: () =>
|
||||
`
|
||||
Expected to be: ${this.utils.printExpected(clearMatch)}
|
||||
Received: ${this.utils.printReceived(clearReceived)}
|
||||
`;
|
||||
|
||||
return { actual: received, message, pass };
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
describe('custom matcher toEqualGraphqlFixture', () => {
|
||||
const key = 'value';
|
||||
const key2 = 'value2';
|
||||
|
||||
describe('positive assertion', () => {
|
||||
it.each([
|
||||
[{}, {}],
|
||||
[{ undef: undefined }, { undef: undefined }],
|
||||
[{}, { __typename: 'MyType' }],
|
||||
[{ key }, { key, __typename: 'MyType' }],
|
||||
[
|
||||
{ obj: { key } },
|
||||
{
|
||||
obj: {
|
||||
key,
|
||||
__typename: 'MyNestedType',
|
||||
},
|
||||
__typename: 'MyType',
|
||||
},
|
||||
],
|
||||
[[{ key }], [{ key }]],
|
||||
[
|
||||
[{ key }],
|
||||
[
|
||||
{
|
||||
key,
|
||||
__typename: 'MyCollectionType',
|
||||
},
|
||||
],
|
||||
],
|
||||
])('%j equals %j', (received, match) => {
|
||||
expect(received).toEqualGraphqlFixture(match);
|
||||
});
|
||||
});
|
||||
|
||||
describe('negative assertion', () => {
|
||||
it.each([
|
||||
[{ __typename: 'MyType' }, {}],
|
||||
[{ key }, { key2, __typename: 'MyType' }],
|
||||
[
|
||||
{ key, key2 },
|
||||
{ key2, __typename: 'MyType' },
|
||||
],
|
||||
[[{ key }, { key2 }], [{ key }]],
|
||||
[
|
||||
[{ key, key2 }],
|
||||
[
|
||||
{
|
||||
key,
|
||||
__typename: 'MyCollectionType',
|
||||
},
|
||||
],
|
||||
],
|
||||
])('%j does not equal %j', (received, match) => {
|
||||
expect(received).not.toEqualGraphqlFixture(match);
|
||||
});
|
||||
});
|
||||
});
|
|
@ -214,7 +214,6 @@ Object {
|
|||
"selfUrl": "http://localhost/releases-namespace/releases-project/-/releases/v1.1",
|
||||
},
|
||||
"assets": Object {
|
||||
"count": undefined,
|
||||
"links": Array [
|
||||
Object {
|
||||
"directAssetPath": "/binaries/awesome-app-3",
|
||||
|
@ -247,29 +246,22 @@ Object {
|
|||
],
|
||||
"sources": Array [],
|
||||
},
|
||||
"author": undefined,
|
||||
"description": "Best. Release. **Ever.** :rocket:",
|
||||
"evidences": Array [],
|
||||
"milestones": Array [
|
||||
Object {
|
||||
"id": "gid://gitlab/Milestone/123",
|
||||
"issueStats": Object {},
|
||||
"stats": undefined,
|
||||
"title": "12.3",
|
||||
"webPath": undefined,
|
||||
"webUrl": undefined,
|
||||
},
|
||||
Object {
|
||||
"id": "gid://gitlab/Milestone/124",
|
||||
"issueStats": Object {},
|
||||
"stats": undefined,
|
||||
"title": "12.4",
|
||||
"webPath": undefined,
|
||||
"webUrl": undefined,
|
||||
},
|
||||
],
|
||||
"name": "The first release",
|
||||
"releasedAt": 2018-12-10T00:00:00.000Z,
|
||||
"releasedAt": "2018-12-10T00:00:00.000Z",
|
||||
"tagName": "v1.1",
|
||||
"tagPath": "/releases-namespace/releases-project/-/tags/v1.1",
|
||||
},
|
||||
|
|
|
@ -7,6 +7,7 @@ import {
|
|||
convertAllReleasesGraphQLResponse,
|
||||
convertOneReleaseGraphQLResponse,
|
||||
} from '~/releases/util';
|
||||
import { stripTypenames } from 'helpers/graphql_helpers';
|
||||
|
||||
describe('releases/util.js', () => {
|
||||
describe('convertGraphQLRelease', () => {
|
||||
|
@ -136,7 +137,7 @@ describe('releases/util.js', () => {
|
|||
describe('convertOneReleaseForEditingGraphQLResponse', () => {
|
||||
it('matches snapshot', () => {
|
||||
expect(
|
||||
convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse),
|
||||
stripTypenames(convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse)),
|
||||
).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
|
@ -87,10 +87,10 @@ describe('AdminRunnerEditApp', () => {
|
|||
await createComponentWithApollo();
|
||||
|
||||
expect(findRunnerUpdateForm().props()).toMatchObject({
|
||||
runner: mockRunner,
|
||||
loading: false,
|
||||
runnerPath: mockRunnerPath,
|
||||
});
|
||||
expect(findRunnerUpdateForm().props('runner')).toEqualGraphqlFixture(mockRunner);
|
||||
});
|
||||
|
||||
describe('When there is an error', () => {
|
||||
|
|
|
@ -256,7 +256,7 @@ describe('AdminRunnerShowApp', () => {
|
|||
await createComponent({ stubs });
|
||||
|
||||
expect(findJobCountBadge().text()).toBe('3');
|
||||
expect(findRunnersJobs().props('runner')).toEqual({ ...mockRunner, ...runner });
|
||||
expect(findRunnersJobs().props('runner')).toEqualGraphqlFixture({ ...mockRunner, ...runner });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -167,7 +167,7 @@ describe('AdminRunnersApp', () => {
|
|||
it('shows the runners list', async () => {
|
||||
await createComponent();
|
||||
|
||||
expect(findRunnerList().props('runners')).toEqual(mockRunners);
|
||||
expect(findRunnerList().props('runners')).toEqualGraphqlFixture(mockRunners);
|
||||
});
|
||||
|
||||
it('runner item links to the runner admin page', async () => {
|
||||
|
@ -188,7 +188,7 @@ describe('AdminRunnersApp', () => {
|
|||
const runnerActions = wrapper.find('tr [data-testid="td-actions"]').find(RunnerActionsCell);
|
||||
const runner = mockRunners[0];
|
||||
|
||||
expect(runnerActions.props()).toEqual({
|
||||
expect(runnerActions.props()).toEqualGraphqlFixture({
|
||||
runner,
|
||||
editUrl: runner.editAdminUrl,
|
||||
});
|
||||
|
|
|
@ -73,8 +73,7 @@ describe('RunnerJobs', () => {
|
|||
it('Shows jobs', () => {
|
||||
const jobs = findRunnerJobsTable().props('jobs');
|
||||
|
||||
expect(jobs).toHaveLength(mockJobs.length);
|
||||
expect(jobs[0]).toMatchObject(mockJobs[0]);
|
||||
expect(jobs).toEqualGraphqlFixture(mockJobs);
|
||||
});
|
||||
|
||||
describe('When "Next" page is clicked', () => {
|
||||
|
|
|
@ -107,7 +107,7 @@ describe('GroupRunnerShowApp', () => {
|
|||
});
|
||||
|
||||
it('renders runner details component', () => {
|
||||
expect(findRunnerDetails().props('runner')).toEqual(mockRunner);
|
||||
expect(findRunnerDetails().props('runner')).toEqualGraphqlFixture(mockRunner);
|
||||
});
|
||||
|
||||
describe('when runner cannot be updated', () => {
|
||||
|
|
|
@ -167,7 +167,7 @@ describe('GroupRunnersApp', () => {
|
|||
await createComponent();
|
||||
|
||||
const runners = findRunnerList().props('runners');
|
||||
expect(runners).toEqual(mockGroupRunnersEdges.map(({ node }) => node));
|
||||
expect(runners).toEqualGraphqlFixture(mockGroupRunnersEdges.map(({ node }) => node));
|
||||
});
|
||||
|
||||
it('requests the runners with group path and no other filters', async () => {
|
||||
|
|
|
@ -84,6 +84,7 @@ RSpec.describe Gitlab::VersionInfo do
|
|||
end
|
||||
|
||||
describe '.parse' do
|
||||
it { expect(described_class.parse(described_class.new(1, 0, 0))).to eq(@v1_0_0) }
|
||||
it { expect(described_class.parse("1.0.0")).to eq(@v1_0_0) }
|
||||
it { expect(described_class.parse("1.0.0.1")).to eq(@v1_0_0) }
|
||||
it { expect(described_class.parse("1.0.0-ee")).to eq(@v1_0_0) }
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
require 'action_dispatch/testing/test_request'
|
||||
require 'fileutils'
|
||||
require 'graphlyte'
|
||||
|
||||
require_relative '../../../lib/gitlab/popen'
|
||||
|
||||
|
@ -47,7 +48,8 @@ module JavaScriptFixturesHelpers
|
|||
path = Rails.root / base / query_path
|
||||
queries = Gitlab::Graphql::Queries.find(path)
|
||||
if queries.length == 1
|
||||
queries.first.text(mode: Gitlab.ee? ? :ee : :ce )
|
||||
query = queries.first.text(mode: Gitlab.ee? ? :ee : :ce )
|
||||
inflate_query_with_typenames(query)
|
||||
else
|
||||
raise "Could not find query file at #{path}, please check your query_path" % path
|
||||
end
|
||||
|
@ -55,6 +57,23 @@ module JavaScriptFixturesHelpers
|
|||
|
||||
private
|
||||
|
||||
# Private: Parse a GraphQL query and inflate the fields with a __typename
|
||||
#
|
||||
# query - the GraqhQL query to parse
|
||||
def inflate_query_with_typenames(query, doc: Graphlyte.parse(query))
|
||||
typename_editor.edit(doc)
|
||||
|
||||
doc.to_s
|
||||
end
|
||||
|
||||
def typename_editor
|
||||
typename = Graphlyte::Syntax::Field.new(name: '__typename')
|
||||
|
||||
@editor ||= Graphlyte::Editor.new.on_field do |field|
|
||||
field.selection << typename unless field.selection.empty? || field.selection.map(&:name).include?('__typename')
|
||||
end
|
||||
end
|
||||
|
||||
# Private: Store a response object as fixture file
|
||||
#
|
||||
# response - string or response object to store
|
||||
|
|
|
@ -17,10 +17,6 @@ workflow:
|
|||
script:
|
||||
- bundle exec rspec
|
||||
|
||||
rspec-2.6:
|
||||
image: "ruby:2.6"
|
||||
extends: .rspec
|
||||
|
||||
rspec-2.7:
|
||||
image: "ruby:2.7"
|
||||
extends: .rspec
|
||||
|
|
|
@ -17,14 +17,10 @@ workflow:
|
|||
script:
|
||||
- bundle exec rspec
|
||||
|
||||
rspec-2.6:
|
||||
image: "ruby:2.6"
|
||||
extends: .rspec
|
||||
|
||||
rspec-2.7:
|
||||
image: "ruby:2.7"
|
||||
extends: .rspec
|
||||
|
||||
rspec-3.0:
|
||||
image: "ruby:3.0"
|
||||
extends: .rspec
|
||||
extends: .rspec
|
||||
|
|
|
@ -17,10 +17,6 @@ workflow:
|
|||
script:
|
||||
- bundle exec rspec
|
||||
|
||||
rspec-2.6:
|
||||
image: "ruby:2.6"
|
||||
extends: .rspec
|
||||
|
||||
rspec-2.7:
|
||||
image: "ruby:2.7"
|
||||
extends: .rspec
|
||||
|
|
|
@ -9,49 +9,51 @@ PATH
|
|||
GEM
|
||||
remote: http://rubygems.org/
|
||||
specs:
|
||||
activesupport (5.0.0.1)
|
||||
activesupport (7.0.3.1)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
i18n (~> 0.7)
|
||||
minitest (~> 5.1)
|
||||
tzinfo (~> 1.1)
|
||||
addressable (2.5.2)
|
||||
public_suffix (>= 2.0.2, < 4.0)
|
||||
concurrent-ruby (1.0.5)
|
||||
crack (0.4.3)
|
||||
safe_yaml (~> 1.0.0)
|
||||
diff-lcs (1.2.5)
|
||||
hashdiff (0.3.6)
|
||||
hashie (3.4.3)
|
||||
i18n (0.8.1)
|
||||
mini_portile2 (2.1.0)
|
||||
minitest (5.10.1)
|
||||
nokogiri (1.6.8.1)
|
||||
mini_portile2 (~> 2.1.0)
|
||||
omniauth (1.3.1)
|
||||
hashie (>= 1.2, < 4)
|
||||
rack (>= 1.0, < 3)
|
||||
public_suffix (3.0.0)
|
||||
rack (1.6.4)
|
||||
rack-test (0.6.3)
|
||||
rack (>= 1.0)
|
||||
rake (10.5.0)
|
||||
i18n (>= 1.6, < 2)
|
||||
minitest (>= 5.1)
|
||||
tzinfo (~> 2.0)
|
||||
addressable (2.8.0)
|
||||
public_suffix (>= 2.0.2, < 5.0)
|
||||
concurrent-ruby (1.1.10)
|
||||
crack (0.4.5)
|
||||
rexml
|
||||
diff-lcs (1.5.0)
|
||||
hashdiff (1.0.1)
|
||||
hashie (5.0.0)
|
||||
i18n (1.12.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
mini_portile2 (2.8.0)
|
||||
minitest (5.16.2)
|
||||
nokogiri (1.13.8)
|
||||
mini_portile2 (~> 2.8.0)
|
||||
racc (~> 1.4)
|
||||
omniauth (1.9.1)
|
||||
hashie (>= 3.4.6)
|
||||
rack (>= 1.6.2, < 3)
|
||||
public_suffix (4.0.7)
|
||||
racc (1.6.0)
|
||||
rack (2.2.4)
|
||||
rack-test (2.0.2)
|
||||
rack (>= 1.3)
|
||||
rake (13.0.6)
|
||||
rexml (3.2.5)
|
||||
rspec (3.0.0)
|
||||
rspec-core (~> 3.0.0)
|
||||
rspec-expectations (~> 3.0.0)
|
||||
rspec-mocks (~> 3.0.0)
|
||||
rspec-core (3.0.4)
|
||||
rspec-support (~> 3.0.0)
|
||||
rspec-expectations (3.0.4)
|
||||
rspec (3.11.0)
|
||||
rspec-core (~> 3.11.0)
|
||||
rspec-expectations (~> 3.11.0)
|
||||
rspec-mocks (~> 3.11.0)
|
||||
rspec-core (3.11.0)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-expectations (3.11.0)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.0.0)
|
||||
rspec-mocks (3.0.4)
|
||||
rspec-support (~> 3.0.0)
|
||||
rspec-support (3.0.4)
|
||||
safe_yaml (1.0.4)
|
||||
thread_safe (0.3.6)
|
||||
tzinfo (1.2.2)
|
||||
thread_safe (~> 0.1)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-mocks (3.11.1)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-support (3.11.0)
|
||||
tzinfo (2.0.5)
|
||||
concurrent-ruby (~> 1.0)
|
||||
webmock (3.0.1)
|
||||
addressable (>= 2.3.6)
|
||||
crack (>= 0.3.2)
|
||||
|
@ -67,8 +69,8 @@ DEPENDENCIES
|
|||
rack-test
|
||||
rake
|
||||
rexml (~> 3.2.5)
|
||||
rspec (~> 3.0.0)
|
||||
rspec (>= 3.4)
|
||||
webmock (~> 3.0.0)
|
||||
|
||||
BUNDLED WITH
|
||||
2.3.15
|
||||
2.3.19
|
||||
|
|
|
@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|||
gem.add_development_dependency(%q<rake>, [">= 0"])
|
||||
gem.add_development_dependency(%q<rack-test>, [">= 0"])
|
||||
gem.add_development_dependency(%q<rexml>, ["~> 3.2.5"])
|
||||
gem.add_development_dependency(%q<rspec>, ["~> 3.0.0"])
|
||||
gem.add_development_dependency(%q<rspec>, [">= 3.4"])
|
||||
gem.add_development_dependency(%q<webmock>, ["~> 3.0.0"])
|
||||
gem.add_development_dependency(%q<bundler>, ["> 1.0.0"])
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue