Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
f805496e2f
commit
2857ba3ce5
|
@ -1,8 +1,5 @@
|
||||||
---
|
---
|
||||||
Naming/HeredocDelimiterNaming:
|
Naming/HeredocDelimiterNaming:
|
||||||
# Offense count: 388
|
|
||||||
# Temporarily disabled due to too many offenses
|
|
||||||
Enabled: false
|
|
||||||
Exclude:
|
Exclude:
|
||||||
- 'app/models/ci/build_trace_chunks/redis_base.rb'
|
- 'app/models/ci/build_trace_chunks/redis_base.rb'
|
||||||
- 'app/models/concerns/counter_attribute.rb'
|
- 'app/models/concerns/counter_attribute.rb'
|
||||||
|
@ -31,6 +28,7 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'ee/spec/services/security/security_orchestration_policies/policy_commit_service_spec.rb'
|
- 'ee/spec/services/security/security_orchestration_policies/policy_commit_service_spec.rb'
|
||||||
- 'ee/spec/support/helpers/ee/ldap_helpers.rb'
|
- 'ee/spec/support/helpers/ee/ldap_helpers.rb'
|
||||||
- 'ee/spec/tasks/gitlab/elastic_rake_spec.rb'
|
- 'ee/spec/tasks/gitlab/elastic_rake_spec.rb'
|
||||||
|
- 'lib/api/metadata.rb'
|
||||||
- 'lib/api/version.rb'
|
- 'lib/api/version.rb'
|
||||||
- 'lib/backup/helper.rb'
|
- 'lib/backup/helper.rb'
|
||||||
- 'lib/feature/shared.rb'
|
- 'lib/feature/shared.rb'
|
||||||
|
@ -46,14 +44,14 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'lib/tasks/gitlab/docs/compile_deprecations.rake'
|
- 'lib/tasks/gitlab/docs/compile_deprecations.rake'
|
||||||
- 'lib/tasks/gitlab/password.rake'
|
- 'lib/tasks/gitlab/password.rake'
|
||||||
- 'qa/qa/scenario/test/sanity/selectors.rb'
|
- 'qa/qa/scenario/test/sanity/selectors.rb'
|
||||||
- 'qa/qa/service/docker_run/gitlab_runner.rb'
|
|
||||||
- 'qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb'
|
|
||||||
- 'qa/qa/specs/features/browser_ui/3_create/web_ide/web_terminal_spec.rb'
|
- 'qa/qa/specs/features/browser_ui/3_create/web_ide/web_terminal_spec.rb'
|
||||||
- 'qa/qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb'
|
- 'qa/qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb'
|
||||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb'
|
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb'
|
||||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb'
|
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb'
|
||||||
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb'
|
- 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb'
|
||||||
- 'rubocop/cop/database/multiple_databases.rb'
|
- 'rubocop/cop/database/multiple_databases.rb'
|
||||||
|
- 'rubocop/cop/database/rescue_query_canceled.rb'
|
||||||
|
- 'rubocop/cop/database/rescue_statement_timeout.rb'
|
||||||
- 'rubocop/cop/default_scope.rb'
|
- 'rubocop/cop/default_scope.rb'
|
||||||
- 'rubocop/cop/file_decompression.rb'
|
- 'rubocop/cop/file_decompression.rb'
|
||||||
- 'rubocop/cop/gitlab/httparty.rb'
|
- 'rubocop/cop/gitlab/httparty.rb'
|
||||||
|
@ -61,6 +59,7 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'rubocop/cop/gitlab/module_with_instance_variables.rb'
|
- 'rubocop/cop/gitlab/module_with_instance_variables.rb'
|
||||||
- 'rubocop/cop/gitlab/predicate_memoization.rb'
|
- 'rubocop/cop/gitlab/predicate_memoization.rb'
|
||||||
- 'spec/controllers/projects/pipelines_controller_spec.rb'
|
- 'spec/controllers/projects/pipelines_controller_spec.rb'
|
||||||
|
- 'spec/db/docs_spec.rb'
|
||||||
- 'spec/deprecation_toolkit_env.rb'
|
- 'spec/deprecation_toolkit_env.rb'
|
||||||
- 'spec/factories/packages/debian/distribution.rb'
|
- 'spec/factories/packages/debian/distribution.rb'
|
||||||
- 'spec/factories/packages/debian/file_metadatum.rb'
|
- 'spec/factories/packages/debian/file_metadatum.rb'
|
||||||
|
@ -68,6 +67,7 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'spec/features/task_lists_spec.rb'
|
- 'spec/features/task_lists_spec.rb'
|
||||||
- 'spec/helpers/markup_helper_spec.rb'
|
- 'spec/helpers/markup_helper_spec.rb'
|
||||||
- 'spec/initializers/100_patch_omniauth_oauth2_spec.rb'
|
- 'spec/initializers/100_patch_omniauth_oauth2_spec.rb'
|
||||||
|
- 'spec/initializers/net_http_response_patch_spec.rb'
|
||||||
- 'spec/initializers/rack_multipart_patch_spec.rb'
|
- 'spec/initializers/rack_multipart_patch_spec.rb'
|
||||||
- 'spec/initializers/secret_token_spec.rb'
|
- 'spec/initializers/secret_token_spec.rb'
|
||||||
- 'spec/initializers/validate_database_config_spec.rb'
|
- 'spec/initializers/validate_database_config_spec.rb'
|
||||||
|
@ -82,6 +82,7 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'spec/lib/gitlab/ci/yaml_processor_spec.rb'
|
- 'spec/lib/gitlab/ci/yaml_processor_spec.rb'
|
||||||
- 'spec/lib/gitlab/cluster/mixins/puma_cluster_spec.rb'
|
- 'spec/lib/gitlab/cluster/mixins/puma_cluster_spec.rb'
|
||||||
- 'spec/lib/gitlab/conflict/file_collection_spec.rb'
|
- 'spec/lib/gitlab/conflict/file_collection_spec.rb'
|
||||||
|
- 'spec/lib/gitlab/database/loose_foreign_keys_spec.rb'
|
||||||
- 'spec/lib/gitlab/diff/file_spec.rb'
|
- 'spec/lib/gitlab/diff/file_spec.rb'
|
||||||
- 'spec/lib/gitlab/diff/pair_selector_spec.rb'
|
- 'spec/lib/gitlab/diff/pair_selector_spec.rb'
|
||||||
- 'spec/lib/gitlab/diff/parser_spec.rb'
|
- 'spec/lib/gitlab/diff/parser_spec.rb'
|
||||||
|
@ -131,10 +132,10 @@ Naming/HeredocDelimiterNaming:
|
||||||
- 'spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb'
|
- 'spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb'
|
||||||
- 'spec/services/google_cloud/generate_pipeline_service_spec.rb'
|
- 'spec/services/google_cloud/generate_pipeline_service_spec.rb'
|
||||||
- 'spec/services/task_list_toggle_service_spec.rb'
|
- 'spec/services/task_list_toggle_service_spec.rb'
|
||||||
- 'spec/support/helpers/seed_helper.rb'
|
|
||||||
- 'spec/support/helpers/stub_object_storage.rb'
|
- 'spec/support/helpers/stub_object_storage.rb'
|
||||||
- 'spec/support/shared_examples/helm_commands_shared_examples.rb'
|
- 'spec/support/shared_examples/helm_commands_shared_examples.rb'
|
||||||
- 'spec/support/shared_examples/models/taskable_shared_examples.rb'
|
- 'spec/support/shared_examples/models/taskable_shared_examples.rb'
|
||||||
- 'spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb'
|
- 'spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb'
|
||||||
- 'spec/support/test_reports/test_reports_helper.rb'
|
- 'spec/support/test_reports/test_reports_helper.rb'
|
||||||
|
- 'spec/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences_rake_spec.rb'
|
||||||
- 'spec/workers/post_receive_spec.rb'
|
- 'spec/workers/post_receive_spec.rb'
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
3d769a33712796de113fe07647597a63d762c305
|
993d944ebeea3e0ec8157481f125f9c70161ae4a
|
||||||
|
|
4
Gemfile
4
Gemfile
|
@ -78,7 +78,7 @@ gem 'u2f', '~> 0.2.1'
|
||||||
gem 'validates_hostname', '~> 1.0.11'
|
gem 'validates_hostname', '~> 1.0.11'
|
||||||
gem 'rubyzip', '~> 2.3.2', require: 'zip'
|
gem 'rubyzip', '~> 2.3.2', require: 'zip'
|
||||||
# GitLab Pages letsencrypt support
|
# GitLab Pages letsencrypt support
|
||||||
gem 'acme-client', '~> 2.0', '>= 2.0.9'
|
gem 'acme-client', '~> 2.0'
|
||||||
|
|
||||||
# Browser detection
|
# Browser detection
|
||||||
gem 'browser', '~> 4.2'
|
gem 'browser', '~> 4.2'
|
||||||
|
@ -435,6 +435,8 @@ group :test do
|
||||||
gem 'capybara-screenshot', '~> 1.0.22'
|
gem 'capybara-screenshot', '~> 1.0.22'
|
||||||
gem 'selenium-webdriver', '~> 3.142'
|
gem 'selenium-webdriver', '~> 3.142'
|
||||||
|
|
||||||
|
gem 'graphlyte', '~> 1.0.0'
|
||||||
|
|
||||||
gem 'shoulda-matchers', '~> 5.1.0', require: false
|
gem 'shoulda-matchers', '~> 5.1.0', require: false
|
||||||
gem 'email_spec', '~> 2.2.0'
|
gem 'email_spec', '~> 2.2.0'
|
||||||
gem 'webmock', '~> 3.9.1'
|
gem 'webmock', '~> 3.9.1'
|
||||||
|
|
|
@ -43,8 +43,9 @@ GEM
|
||||||
remote: https://rubygems.org/
|
remote: https://rubygems.org/
|
||||||
specs:
|
specs:
|
||||||
RedCloth (4.3.2)
|
RedCloth (4.3.2)
|
||||||
acme-client (2.0.9)
|
acme-client (2.0.11)
|
||||||
faraday (>= 0.17, < 2.0.0)
|
faraday (>= 1.0, < 3.0.0)
|
||||||
|
faraday-retry (~> 1.0)
|
||||||
actioncable (6.1.6.1)
|
actioncable (6.1.6.1)
|
||||||
actionpack (= 6.1.6.1)
|
actionpack (= 6.1.6.1)
|
||||||
activesupport (= 6.1.6.1)
|
activesupport (= 6.1.6.1)
|
||||||
|
@ -615,6 +616,7 @@ GEM
|
||||||
faraday (>= 1.0)
|
faraday (>= 1.0)
|
||||||
faraday_middleware
|
faraday_middleware
|
||||||
graphql-client
|
graphql-client
|
||||||
|
graphlyte (1.0.0)
|
||||||
graphql (1.13.12)
|
graphql (1.13.12)
|
||||||
graphql-client (0.17.0)
|
graphql-client (0.17.0)
|
||||||
activesupport (>= 3.0)
|
activesupport (>= 3.0)
|
||||||
|
@ -1481,7 +1483,7 @@ PLATFORMS
|
||||||
|
|
||||||
DEPENDENCIES
|
DEPENDENCIES
|
||||||
RedCloth (~> 4.3.2)
|
RedCloth (~> 4.3.2)
|
||||||
acme-client (~> 2.0, >= 2.0.9)
|
acme-client (~> 2.0)
|
||||||
activerecord-explain-analyze (~> 0.1)
|
activerecord-explain-analyze (~> 0.1)
|
||||||
acts-as-taggable-on (~> 9.0)
|
acts-as-taggable-on (~> 9.0)
|
||||||
addressable (~> 2.8)
|
addressable (~> 2.8)
|
||||||
|
@ -1593,6 +1595,7 @@ DEPENDENCIES
|
||||||
grape_logging (~> 1.8)
|
grape_logging (~> 1.8)
|
||||||
graphiql-rails (~> 1.8)
|
graphiql-rails (~> 1.8)
|
||||||
graphlient (~> 0.5.0)
|
graphlient (~> 0.5.0)
|
||||||
|
graphlyte (~> 1.0.0)
|
||||||
graphql (~> 1.13.12)
|
graphql (~> 1.13.12)
|
||||||
graphql-docs (~> 2.1.0)
|
graphql-docs (~> 2.1.0)
|
||||||
grpc (~> 1.42.0)
|
grpc (~> 1.42.0)
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
fragment RunnerFieldsShared on CiRunner {
|
fragment RunnerFieldsShared on CiRunner {
|
||||||
__typename
|
|
||||||
id
|
id
|
||||||
shortSha
|
shortSha
|
||||||
runnerType
|
runnerType
|
||||||
|
|
|
@ -31,7 +31,6 @@ query getAllRunners(
|
||||||
editAdminUrl
|
editAdminUrl
|
||||||
}
|
}
|
||||||
pageInfo {
|
pageInfo {
|
||||||
__typename
|
|
||||||
...PageInfo
|
...PageInfo
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,6 @@ query getGroupRunners(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pageInfo {
|
pageInfo {
|
||||||
__typename
|
|
||||||
...PageInfo
|
...PageInfo
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
fragment ListItemShared on CiRunner {
|
fragment ListItemShared on CiRunner {
|
||||||
__typename
|
|
||||||
id
|
id
|
||||||
description
|
description
|
||||||
runnerType
|
runnerType
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
fragment RunnerDetailsShared on CiRunner {
|
fragment RunnerDetailsShared on CiRunner {
|
||||||
__typename
|
|
||||||
id
|
id
|
||||||
shortSha
|
shortSha
|
||||||
runnerType
|
runnerType
|
||||||
|
|
|
@ -41,7 +41,7 @@
|
||||||
= n_('%s additional commit has been omitted to prevent performance issues.', '%s additional commits have been omitted to prevent performance issues.', hidden) % number_with_delimiter(hidden)
|
= n_('%s additional commit has been omitted to prevent performance issues.', '%s additional commits have been omitted to prevent performance issues.', hidden) % number_with_delimiter(hidden)
|
||||||
|
|
||||||
- if can_update_merge_request && context_commits&.empty?
|
- if can_update_merge_request && context_commits&.empty?
|
||||||
= render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5', data: { context_commits_empty: 'true' } }) do
|
= render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5 add-review-item-modal-trigger', data: { context_commits_empty: 'true' } }) do
|
||||||
= _('Add previously merged commits')
|
= _('Add previously merged commits')
|
||||||
|
|
||||||
- if commits.size == 0 && context_commits.nil?
|
- if commits.size == 0 && context_commits.nil?
|
||||||
|
|
|
@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/364101
|
||||||
milestone: '15.1'
|
milestone: '15.1'
|
||||||
type: development
|
type: development
|
||||||
group: group::source code
|
group: group::source code
|
||||||
default_enabled: false
|
default_enabled: true
|
||||||
|
|
|
@ -0,0 +1,15 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class IndexPersonalAccessTokensOnIdAndCreatedAt < Gitlab::Database::Migration[2.0]
|
||||||
|
INDEX_NAME = 'index_personal_access_tokens_on_id_and_created_at'
|
||||||
|
|
||||||
|
disable_ddl_transaction!
|
||||||
|
|
||||||
|
def up
|
||||||
|
add_concurrent_index :personal_access_tokens, [:id, :created_at], name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
remove_concurrent_index_by_name :personal_access_tokens, INDEX_NAME
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1 @@
|
||||||
|
b0499c9b4cf3f39eec49dc7def7eaf8f1bbd03f2a34ba9eefa8440a109672136
|
|
@ -29251,6 +29251,8 @@ CREATE INDEX index_path_locks_on_user_id ON path_locks USING btree (user_id);
|
||||||
|
|
||||||
CREATE INDEX index_pe_approval_rules_on_required_approvals_and_created_at ON protected_environment_approval_rules USING btree (required_approvals, created_at);
|
CREATE INDEX index_pe_approval_rules_on_required_approvals_and_created_at ON protected_environment_approval_rules USING btree (required_approvals, created_at);
|
||||||
|
|
||||||
|
CREATE INDEX index_personal_access_tokens_on_id_and_created_at ON personal_access_tokens USING btree (id, created_at);
|
||||||
|
|
||||||
CREATE UNIQUE INDEX index_personal_access_tokens_on_token_digest ON personal_access_tokens USING btree (token_digest);
|
CREATE UNIQUE INDEX index_personal_access_tokens_on_token_digest ON personal_access_tokens USING btree (token_digest);
|
||||||
|
|
||||||
CREATE INDEX index_personal_access_tokens_on_user_id ON personal_access_tokens USING btree (user_id);
|
CREATE INDEX index_personal_access_tokens_on_user_id ON personal_access_tokens USING btree (user_id);
|
||||||
|
|
|
@ -1048,7 +1048,8 @@ The [secure files API](../api/secure_files.md) enforces the following limits:
|
||||||
|
|
||||||
## Changelog API limits
|
## Changelog API limits
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default.
|
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default.
|
||||||
|
> - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/33893) in GitLab 15.3.
|
||||||
|
|
||||||
The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits:
|
The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits:
|
||||||
|
|
||||||
|
|
|
@ -384,7 +384,7 @@ POST /projects/:id/releases
|
||||||
| `assets:links` | array of hash | no | An array of assets links. |
|
| `assets:links` | array of hash | no | An array of assets links. |
|
||||||
| `assets:links:name`| string | required by: `assets:links` | The name of the link. Link names must be unique within the release. |
|
| `assets:links:name`| string | required by: `assets:links` | The name of the link. Link names must be unique within the release. |
|
||||||
| `assets:links:url` | string | required by: `assets:links` | The URL of the link. Link URLs must be unique within the release. |
|
| `assets:links:url` | string | required by: `assets:links` | The URL of the link. Link URLs must be unique within the release. |
|
||||||
| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets).
|
| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets).
|
||||||
| `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`.
|
| `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`.
|
||||||
| `released_at` | datetime | no | The date when the release is/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). |
|
| `released_at` | datetime | no | The date when the release is/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). |
|
||||||
|
|
||||||
|
|
|
@ -93,14 +93,14 @@ Create an asset as a link from a Release.
|
||||||
POST /projects/:id/releases/:tag_name/assets/links
|
POST /projects/:id/releases/:tag_name/assets/links
|
||||||
```
|
```
|
||||||
|
|
||||||
| Attribute | Type | Required | Description |
|
| Attribute | Type | Required | Description |
|
||||||
| ------------- | -------------- | -------- | ---------------------------------------------------------------------------------------------------------------- |
|
|-------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). |
|
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). |
|
||||||
| `tag_name` | string | yes | The tag associated with the Release. |
|
| `tag_name` | string | yes | The tag associated with the Release. |
|
||||||
| `name` | string | yes | The name of the link. Link names must be unique within the release. |
|
| `name` | string | yes | The name of the link. Link names must be unique in the release. |
|
||||||
| `url` | string | yes | The URL of the link. Link URLs must be unique within the release. |
|
| `url` | string | yes | The URL of the link. Link URLs must be unique in the release. |
|
||||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). |
|
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). |
|
||||||
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
||||||
|
|
||||||
Example request:
|
Example request:
|
||||||
|
|
||||||
|
@ -141,7 +141,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id
|
||||||
| `link_id` | integer | yes | The ID of the link. |
|
| `link_id` | integer | yes | The ID of the link. |
|
||||||
| `name` | string | no | The name of the link. |
|
| `name` | string | no | The name of the link. |
|
||||||
| `url` | string | no | The URL of the link. |
|
| `url` | string | no | The URL of the link. |
|
||||||
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets).
|
| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets).
|
||||||
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. |
|
||||||
|
|
||||||
NOTE:
|
NOTE:
|
||||||
|
|
|
@ -136,7 +136,7 @@ As such, the target size of a physical table after refactoring depends on the si
|
||||||
There is no standard solution to reduce table sizes - there are many!
|
There is no standard solution to reduce table sizes - there are many!
|
||||||
|
|
||||||
1. **Retention**: Delete unnecessary data, for example expire old and unneeded records.
|
1. **Retention**: Delete unnecessary data, for example expire old and unneeded records.
|
||||||
1. **Remove STI**: We still use [single-table inheritance](../../../development/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables.
|
1. **Remove STI**: We still use [single-table inheritance](../../../development/database/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables.
|
||||||
1. **Index optimization**: Drop unnecessary indexes and consolidate overlapping indexes if possible.
|
1. **Index optimization**: Drop unnecessary indexes and consolidate overlapping indexes if possible.
|
||||||
1. **Optimise data types**: Review data type decisions and optimise data types where possible (example: use integer instead of text for an enum column)
|
1. **Optimise data types**: Review data type decisions and optimise data types where possible (example: use integer instead of text for an enum column)
|
||||||
1. **Partitioning**: Apply a partitioning scheme if there is a common access dimension.
|
1. **Partitioning**: Apply a partitioning scheme if there is a common access dimension.
|
||||||
|
|
|
@ -2991,7 +2991,7 @@ released_at: '2021-03-15T08:00:00Z'
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271454) in GitLab 13.12.
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271454) in GitLab 13.12.
|
||||||
|
|
||||||
Use `release:assets:links` to include [asset links](../../user/project/releases/index.md#release-assets) in the release.
|
Use `release:assets:links` to include [asset links](../../user/project/releases/release_fields.md#release-assets) in the release.
|
||||||
|
|
||||||
Requires `release-cli` version v0.4.0 or later.
|
Requires `release-cli` version v0.4.0 or later.
|
||||||
|
|
||||||
|
|
|
@ -1,410 +1,11 @@
|
||||||
---
|
---
|
||||||
stage: Data Stores
|
redirect_to: 'database/adding_database_indexes.md'
|
||||||
group: Database
|
remove_date: '2022-11-05'
|
||||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# Adding Database Indexes
|
This document was moved to [another location](database/adding_database_indexes.md).
|
||||||
|
|
||||||
Indexes can be used to speed up database queries, but when should you add a new
|
<!-- This redirect file can be deleted after <2022-11-05>. -->
|
||||||
index? Traditionally the answer to this question has been to add an index for
|
<!-- Redirects that point to other docs in the same project expire in three months. -->
|
||||||
every column used for filtering or joining data. For example, consider the
|
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
|
||||||
following query:
|
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
|
||||||
|
|
||||||
```sql
|
|
||||||
SELECT *
|
|
||||||
FROM projects
|
|
||||||
WHERE user_id = 2;
|
|
||||||
```
|
|
||||||
|
|
||||||
Here we are filtering by the `user_id` column and as such a developer may decide
|
|
||||||
to index this column.
|
|
||||||
|
|
||||||
While in certain cases indexing columns using the above approach may make sense,
|
|
||||||
it can actually have a negative impact. Whenever you write data to a table, any
|
|
||||||
existing indexes must also be updated. The more indexes there are, the slower this
|
|
||||||
can potentially become. Indexes can also take up significant disk space, depending
|
|
||||||
on the amount of data indexed and the index type. For example, PostgreSQL offers
|
|
||||||
`GIN` indexes which can be used to index certain data types that cannot be
|
|
||||||
indexed by regular B-tree indexes. These indexes, however, generally take up more
|
|
||||||
data and are slower to update compared to B-tree indexes.
|
|
||||||
|
|
||||||
Because of all this, it's important make the following considerations
|
|
||||||
when adding a new index:
|
|
||||||
|
|
||||||
1. Do the new queries re-use as many existing indexes as possible?
|
|
||||||
1. Is there enough data that using an index is faster than iterating over
|
|
||||||
rows in the table?
|
|
||||||
1. Is the overhead of maintaining the index worth the reduction in query
|
|
||||||
timings?
|
|
||||||
|
|
||||||
## Re-using Queries
|
|
||||||
|
|
||||||
The first step is to make sure your query re-uses as many existing indexes as
|
|
||||||
possible. For example, consider the following query:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
SELECT *
|
|
||||||
FROM todos
|
|
||||||
WHERE user_id = 123
|
|
||||||
AND state = 'open';
|
|
||||||
```
|
|
||||||
|
|
||||||
Now imagine we already have an index on the `user_id` column but not on the
|
|
||||||
`state` column. One may think this query performs badly due to `state` being
|
|
||||||
unindexed. In reality the query may perform just fine given the index on
|
|
||||||
`user_id` can filter out enough rows.
|
|
||||||
|
|
||||||
The best way to determine if indexes are re-used is to run your query using
|
|
||||||
`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering,
|
|
||||||
you may find an extra index doesn't make much, if any, difference.
|
|
||||||
|
|
||||||
In short:
|
|
||||||
|
|
||||||
1. Try to write your query in such a way that it re-uses as many existing
|
|
||||||
indexes as possible.
|
|
||||||
1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most
|
|
||||||
ideal query.
|
|
||||||
|
|
||||||
## Data Size
|
|
||||||
|
|
||||||
A database may not use an index even when a regular sequence scan
|
|
||||||
(iterating over all rows) is faster, especially for small tables.
|
|
||||||
|
|
||||||
Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows.
|
|
||||||
You may _not_ want to add an index if the table size is small (<`1,000` records),
|
|
||||||
or if existing indexes already filter out enough rows.
|
|
||||||
|
|
||||||
## Maintenance Overhead
|
|
||||||
|
|
||||||
Indexes have to be updated on every table write. In the case of PostgreSQL, _all_
|
|
||||||
existing indexes are updated whenever data is written to a table. As a
|
|
||||||
result, having many indexes on the same table slows down writes. It's therefore important
|
|
||||||
to balance query performance with the overhead of maintaining an extra index.
|
|
||||||
|
|
||||||
Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases
|
|
||||||
INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth
|
|
||||||
it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE
|
|
||||||
timings are unaffected.
|
|
||||||
|
|
||||||
## Finding Unused Indexes
|
|
||||||
|
|
||||||
To see which indexes are unused you can run the following query:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass))
|
|
||||||
FROM pg_stat_all_indexes
|
|
||||||
WHERE schemaname = 'public'
|
|
||||||
AND idx_scan = 0
|
|
||||||
AND idx_tup_read = 0
|
|
||||||
AND idx_tup_fetch = 0
|
|
||||||
ORDER BY pg_relation_size(indexrelname::regclass) desc;
|
|
||||||
```
|
|
||||||
|
|
||||||
This query outputs a list containing all indexes that are never used and sorts
|
|
||||||
them by indexes sizes in descending order. This query helps in
|
|
||||||
determining whether existing indexes are still required. More information on
|
|
||||||
the meaning of the various columns can be found at
|
|
||||||
<https://www.postgresql.org/docs/current/monitoring-stats.html>.
|
|
||||||
|
|
||||||
To determine if an index is still being used on production, use the following
|
|
||||||
Thanos query with your index name:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m]))
|
|
||||||
```
|
|
||||||
|
|
||||||
Because the query output relies on the actual usage of your database, it
|
|
||||||
may be affected by factors such as:
|
|
||||||
|
|
||||||
- Certain queries never being executed, thus not being able to use certain
|
|
||||||
indexes.
|
|
||||||
- Certain tables having little data, resulting in PostgreSQL using sequence
|
|
||||||
scans instead of index scans.
|
|
||||||
|
|
||||||
This data is only reliable for a frequently used database with
|
|
||||||
plenty of data, and using as many GitLab features as possible.
|
|
||||||
|
|
||||||
## Requirements for naming indexes
|
|
||||||
|
|
||||||
Indexes with complex definitions must be explicitly named rather than
|
|
||||||
relying on the implicit naming behavior of migration methods. In short,
|
|
||||||
that means you **must** provide an explicit name argument for an index
|
|
||||||
created with one or more of the following options:
|
|
||||||
|
|
||||||
- `where`
|
|
||||||
- `using`
|
|
||||||
- `order`
|
|
||||||
- `length`
|
|
||||||
- `type`
|
|
||||||
- `opclass`
|
|
||||||
|
|
||||||
### Considerations for index names
|
|
||||||
|
|
||||||
Check our [Constraints naming conventions](database/constraint_naming_convention.md) page.
|
|
||||||
|
|
||||||
### Why explicit names are required
|
|
||||||
|
|
||||||
As Rails is database agnostic, it generates an index name only
|
|
||||||
from the required options of all indexes: table name and column names.
|
|
||||||
For example, imagine the following two indexes are created in a migration:
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
def up
|
|
||||||
add_index :my_table, :my_column
|
|
||||||
|
|
||||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
Creation of the second index would fail, because Rails would generate
|
|
||||||
the same name for both indexes.
|
|
||||||
|
|
||||||
This naming issue is further complicated by the behavior of the `index_exists?` method.
|
|
||||||
It considers only the table name, column names, and uniqueness specification
|
|
||||||
of the index when making a comparison. Consider:
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
def up
|
|
||||||
unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL')
|
|
||||||
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
|
||||||
end
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
The call to `index_exists?` returns true if **any** index exists on
|
|
||||||
`:my_table` and `:my_column`, and index creation is bypassed.
|
|
||||||
|
|
||||||
The `add_concurrent_index` helper is a requirement for creating indexes
|
|
||||||
on populated tables. Because it cannot be used inside a transactional
|
|
||||||
migration, it has a built-in check that detects if the index already
|
|
||||||
exists. In the event a match is found, index creation is skipped.
|
|
||||||
Without an explicit name argument, Rails can return a false positive
|
|
||||||
for `index_exists?`, causing a required index to not be created
|
|
||||||
properly. By always requiring a name for certain types of indexes, the
|
|
||||||
chance of error is greatly reduced.
|
|
||||||
|
|
||||||
## Temporary indexes
|
|
||||||
|
|
||||||
There may be times when an index is only needed temporarily.
|
|
||||||
|
|
||||||
For example, in a migration, a column of a table might be conditionally
|
|
||||||
updated. To query which columns must be updated in the
|
|
||||||
[query performance guidelines](query_performance.md), an index is needed
|
|
||||||
that would otherwise not be used.
|
|
||||||
|
|
||||||
In these cases, consider a temporary index. To specify a
|
|
||||||
temporary index:
|
|
||||||
|
|
||||||
1. Prefix the index name with `tmp_` and follow the [naming conventions](database/constraint_naming_convention.md).
|
|
||||||
1. Create a follow-up issue to remove the index in the next (or future) milestone.
|
|
||||||
1. Add a comment in the migration mentioning the removal issue.
|
|
||||||
|
|
||||||
A temporary migration would look like:
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled'
|
|
||||||
|
|
||||||
def up
|
|
||||||
# Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234
|
|
||||||
add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME
|
|
||||||
end
|
|
||||||
|
|
||||||
def down
|
|
||||||
remove_concurrent_index_by_name :projects, INDEX_NAME
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
## Create indexes asynchronously
|
|
||||||
|
|
||||||
For very large tables, index creation can be a challenge to manage.
|
|
||||||
While `add_concurrent_index` creates indexes in a way that does not block
|
|
||||||
normal traffic, it can still be problematic when index creation runs for
|
|
||||||
many hours. Necessary database operations like `autovacuum` cannot run, and
|
|
||||||
on GitLab.com, the deployment process is blocked waiting for index
|
|
||||||
creation to finish.
|
|
||||||
|
|
||||||
To limit impact on GitLab.com, a process exists to create indexes
|
|
||||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
|
||||||
index creation can proceed at a lower level of risk.
|
|
||||||
|
|
||||||
### Schedule index creation for a low-impact time
|
|
||||||
|
|
||||||
1. [Schedule the index to be created](#schedule-the-index-to-be-created).
|
|
||||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
|
||||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
|
||||||
|
|
||||||
### Schedule the index to be created
|
|
||||||
|
|
||||||
Create an MR with a post-deployment migration which prepares the index
|
|
||||||
for asynchronous creation. An example of creating an index using
|
|
||||||
the asynchronous index helpers can be seen in the block below. This migration
|
|
||||||
enters the index name and definition into the `postgres_async_indexes`
|
|
||||||
table. The process that runs on weekends pulls indexes from this
|
|
||||||
table and attempt to create them.
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
# in db/post_migrate/
|
|
||||||
|
|
||||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
|
||||||
|
|
||||||
def up
|
|
||||||
prepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
|
|
||||||
def down
|
|
||||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
### Verify the MR was deployed and the index exists in production
|
|
||||||
|
|
||||||
You can verify if the post-deploy migration was executed on GitLab.com by:
|
|
||||||
|
|
||||||
- Executing `/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
|
|
||||||
the post-deploy migration has been executed in the production database. More details in this
|
|
||||||
[guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
|
|
||||||
- Use a meta-command in #database-lab, such as: `\d <index_name>`.
|
|
||||||
- Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
|
||||||
- Ask someone in #database to check if the index exists.
|
|
||||||
- With proper access, you can also verify directly on production or in a
|
|
||||||
production clone.
|
|
||||||
|
|
||||||
### Add a migration to create the index synchronously
|
|
||||||
|
|
||||||
After the index is verified to exist on the production database, create a second
|
|
||||||
merge request that adds the index synchronously. The schema changes must be
|
|
||||||
updated and committed to `structure.sql` in this second merge request.
|
|
||||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
|
||||||
migration as expected for other installations. The below block
|
|
||||||
demonstrates how to create the second migration for the previous
|
|
||||||
asynchronous example.
|
|
||||||
|
|
||||||
**WARNING:**
|
|
||||||
Verify that the index exists in production before merging a second migration with `add_concurrent_index`.
|
|
||||||
If the second migration is deployed before the index has been created,
|
|
||||||
the index is created synchronously when the second migration executes.
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
# in db/post_migrate/
|
|
||||||
|
|
||||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
|
||||||
|
|
||||||
disable_ddl_transaction!
|
|
||||||
|
|
||||||
def up
|
|
||||||
add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
|
|
||||||
def down
|
|
||||||
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
## Test database index changes locally
|
|
||||||
|
|
||||||
You must test the database index changes locally before creating a merge request.
|
|
||||||
|
|
||||||
### Verify indexes created asynchronously
|
|
||||||
|
|
||||||
Use the asynchronous index helpers on your local environment to test changes for creating an index:
|
|
||||||
|
|
||||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
|
||||||
1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table.
|
|
||||||
1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously.
|
|
||||||
1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists.
|
|
||||||
|
|
||||||
## Drop indexes asynchronously
|
|
||||||
|
|
||||||
For very large tables, index destruction can be a challenge to manage.
|
|
||||||
While `remove_concurrent_index` removes indexes in a way that does not block
|
|
||||||
normal traffic, it can still be problematic if index destruction runs for
|
|
||||||
during autovacuum. Necessary database operations like `autovacuum` cannot run, and
|
|
||||||
the deployment process on GitLab.com is blocked while waiting for index
|
|
||||||
destruction to finish.
|
|
||||||
|
|
||||||
To limit the impact on GitLab.com, use the following process to remove indexes
|
|
||||||
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
|
||||||
index destruction can proceed at a lower level of risk.
|
|
||||||
|
|
||||||
1. [Schedule the index to be removed](#schedule-the-index-to-be-removed).
|
|
||||||
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
|
||||||
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
|
||||||
|
|
||||||
### Schedule the index to be removed
|
|
||||||
|
|
||||||
Create an MR with a post-deployment migration which prepares the index
|
|
||||||
for asynchronous destruction. For example. to destroy an index using
|
|
||||||
the asynchronous index helpers:
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
# in db/post_migrate/
|
|
||||||
|
|
||||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
|
||||||
|
|
||||||
def up
|
|
||||||
prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
|
|
||||||
def down
|
|
||||||
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
This migration enters the index name and definition into the `postgres_async_indexes`
|
|
||||||
table. The process that runs on weekends pulls indexes from this table and attempt
|
|
||||||
to remove them.
|
|
||||||
|
|
||||||
You must test the database index changes locally before creating a merge request.
|
|
||||||
|
|
||||||
### Verify the MR was deployed and the index exists in production
|
|
||||||
|
|
||||||
You can verify if the MR was deployed to GitLab.com with
|
|
||||||
`/chatops run auto_deploy status <merge_sha>`. To verify the existence of
|
|
||||||
the index, you can:
|
|
||||||
|
|
||||||
- Use a meta-command in `#database-lab`, for example: `\d <index_name>`.
|
|
||||||
- Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
|
||||||
- Ask someone in `#database` to check if the index exists.
|
|
||||||
- If you have access, you can verify directly on production or in a
|
|
||||||
production clone.
|
|
||||||
|
|
||||||
### Add a migration to destroy the index synchronously
|
|
||||||
|
|
||||||
After you verify the index exists in the production database, create a second
|
|
||||||
merge request that removes the index synchronously. The schema changes must be
|
|
||||||
updated and committed to `structure.sql` in this second merge request.
|
|
||||||
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
|
||||||
migration as expected for other installations. For example, to
|
|
||||||
create the second migration for the previous asynchronous example:
|
|
||||||
|
|
||||||
**WARNING:**
|
|
||||||
Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`.
|
|
||||||
If the second migration is deployed before the index has been destroyed,
|
|
||||||
the index is destroyed synchronously when the second migration executes.
|
|
||||||
|
|
||||||
```ruby
|
|
||||||
# in db/post_migrate/
|
|
||||||
|
|
||||||
INDEX_NAME = 'index_ci_builds_on_some_column'
|
|
||||||
|
|
||||||
disable_ddl_transaction!
|
|
||||||
|
|
||||||
def up
|
|
||||||
remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME
|
|
||||||
end
|
|
||||||
|
|
||||||
def down
|
|
||||||
add_concurrent_index :ci_builds, :some_column, INDEX_NAME
|
|
||||||
end
|
|
||||||
```
|
|
||||||
|
|
||||||
### Verify indexes removed asynchronously
|
|
||||||
|
|
||||||
To test changes for removing an index, use the asynchronous index helpers on your local environment:
|
|
||||||
|
|
||||||
1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
|
||||||
1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table.
|
|
||||||
1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously.
|
|
||||||
1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
|
|
||||||
command `gdk psql` and run `\d <index_name>` to check that the destroyed index no longer exists.
|
|
||||||
|
|
|
@ -0,0 +1,410 @@
|
||||||
|
---
|
||||||
|
stage: Data Stores
|
||||||
|
group: Database
|
||||||
|
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||||
|
---
|
||||||
|
|
||||||
|
# Adding Database Indexes
|
||||||
|
|
||||||
|
Indexes can be used to speed up database queries, but when should you add a new
|
||||||
|
index? Traditionally the answer to this question has been to add an index for
|
||||||
|
every column used for filtering or joining data. For example, consider the
|
||||||
|
following query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT *
|
||||||
|
FROM projects
|
||||||
|
WHERE user_id = 2;
|
||||||
|
```
|
||||||
|
|
||||||
|
Here we are filtering by the `user_id` column and as such a developer may decide
|
||||||
|
to index this column.
|
||||||
|
|
||||||
|
While in certain cases indexing columns using the above approach may make sense,
|
||||||
|
it can actually have a negative impact. Whenever you write data to a table, any
|
||||||
|
existing indexes must also be updated. The more indexes there are, the slower this
|
||||||
|
can potentially become. Indexes can also take up significant disk space, depending
|
||||||
|
on the amount of data indexed and the index type. For example, PostgreSQL offers
|
||||||
|
`GIN` indexes which can be used to index certain data types that cannot be
|
||||||
|
indexed by regular B-tree indexes. These indexes, however, generally take up more
|
||||||
|
data and are slower to update compared to B-tree indexes.
|
||||||
|
|
||||||
|
Because of all this, it's important make the following considerations
|
||||||
|
when adding a new index:
|
||||||
|
|
||||||
|
1. Do the new queries re-use as many existing indexes as possible?
|
||||||
|
1. Is there enough data that using an index is faster than iterating over
|
||||||
|
rows in the table?
|
||||||
|
1. Is the overhead of maintaining the index worth the reduction in query
|
||||||
|
timings?
|
||||||
|
|
||||||
|
## Re-using Queries
|
||||||
|
|
||||||
|
The first step is to make sure your query re-uses as many existing indexes as
|
||||||
|
possible. For example, consider the following query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT *
|
||||||
|
FROM todos
|
||||||
|
WHERE user_id = 123
|
||||||
|
AND state = 'open';
|
||||||
|
```
|
||||||
|
|
||||||
|
Now imagine we already have an index on the `user_id` column but not on the
|
||||||
|
`state` column. One may think this query performs badly due to `state` being
|
||||||
|
unindexed. In reality the query may perform just fine given the index on
|
||||||
|
`user_id` can filter out enough rows.
|
||||||
|
|
||||||
|
The best way to determine if indexes are re-used is to run your query using
|
||||||
|
`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering,
|
||||||
|
you may find an extra index doesn't make much, if any, difference.
|
||||||
|
|
||||||
|
In short:
|
||||||
|
|
||||||
|
1. Try to write your query in such a way that it re-uses as many existing
|
||||||
|
indexes as possible.
|
||||||
|
1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most
|
||||||
|
ideal query.
|
||||||
|
|
||||||
|
## Data Size
|
||||||
|
|
||||||
|
A database may not use an index even when a regular sequence scan
|
||||||
|
(iterating over all rows) is faster, especially for small tables.
|
||||||
|
|
||||||
|
Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows.
|
||||||
|
You may _not_ want to add an index if the table size is small (<`1,000` records),
|
||||||
|
or if existing indexes already filter out enough rows.
|
||||||
|
|
||||||
|
## Maintenance Overhead
|
||||||
|
|
||||||
|
Indexes have to be updated on every table write. In the case of PostgreSQL, _all_
|
||||||
|
existing indexes are updated whenever data is written to a table. As a
|
||||||
|
result, having many indexes on the same table slows down writes. It's therefore important
|
||||||
|
to balance query performance with the overhead of maintaining an extra index.
|
||||||
|
|
||||||
|
Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases
|
||||||
|
INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth
|
||||||
|
it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE
|
||||||
|
timings are unaffected.
|
||||||
|
|
||||||
|
## Finding Unused Indexes
|
||||||
|
|
||||||
|
To see which indexes are unused you can run the following query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass))
|
||||||
|
FROM pg_stat_all_indexes
|
||||||
|
WHERE schemaname = 'public'
|
||||||
|
AND idx_scan = 0
|
||||||
|
AND idx_tup_read = 0
|
||||||
|
AND idx_tup_fetch = 0
|
||||||
|
ORDER BY pg_relation_size(indexrelname::regclass) desc;
|
||||||
|
```
|
||||||
|
|
||||||
|
This query outputs a list containing all indexes that are never used and sorts
|
||||||
|
them by indexes sizes in descending order. This query helps in
|
||||||
|
determining whether existing indexes are still required. More information on
|
||||||
|
the meaning of the various columns can be found at
|
||||||
|
<https://www.postgresql.org/docs/current/monitoring-stats.html>.
|
||||||
|
|
||||||
|
To determine if an index is still being used on production, use the following
|
||||||
|
Thanos query with your index name:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m]))
|
||||||
|
```
|
||||||
|
|
||||||
|
Because the query output relies on the actual usage of your database, it
|
||||||
|
may be affected by factors such as:
|
||||||
|
|
||||||
|
- Certain queries never being executed, thus not being able to use certain
|
||||||
|
indexes.
|
||||||
|
- Certain tables having little data, resulting in PostgreSQL using sequence
|
||||||
|
scans instead of index scans.
|
||||||
|
|
||||||
|
This data is only reliable for a frequently used database with
|
||||||
|
plenty of data, and using as many GitLab features as possible.
|
||||||
|
|
||||||
|
## Requirements for naming indexes
|
||||||
|
|
||||||
|
Indexes with complex definitions must be explicitly named rather than
|
||||||
|
relying on the implicit naming behavior of migration methods. In short,
|
||||||
|
that means you **must** provide an explicit name argument for an index
|
||||||
|
created with one or more of the following options:
|
||||||
|
|
||||||
|
- `where`
|
||||||
|
- `using`
|
||||||
|
- `order`
|
||||||
|
- `length`
|
||||||
|
- `type`
|
||||||
|
- `opclass`
|
||||||
|
|
||||||
|
### Considerations for index names
|
||||||
|
|
||||||
|
Check our [Constraints naming conventions](constraint_naming_convention.md) page.
|
||||||
|
|
||||||
|
### Why explicit names are required
|
||||||
|
|
||||||
|
As Rails is database agnostic, it generates an index name only
|
||||||
|
from the required options of all indexes: table name and column names.
|
||||||
|
For example, imagine the following two indexes are created in a migration:
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
def up
|
||||||
|
add_index :my_table, :my_column
|
||||||
|
|
||||||
|
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
Creation of the second index would fail, because Rails would generate
|
||||||
|
the same name for both indexes.
|
||||||
|
|
||||||
|
This naming issue is further complicated by the behavior of the `index_exists?` method.
|
||||||
|
It considers only the table name, column names, and uniqueness specification
|
||||||
|
of the index when making a comparison. Consider:
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
def up
|
||||||
|
unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL')
|
||||||
|
add_index :my_table, :my_column, where: 'my_column IS NOT NULL'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
The call to `index_exists?` returns true if **any** index exists on
|
||||||
|
`:my_table` and `:my_column`, and index creation is bypassed.
|
||||||
|
|
||||||
|
The `add_concurrent_index` helper is a requirement for creating indexes
|
||||||
|
on populated tables. Because it cannot be used inside a transactional
|
||||||
|
migration, it has a built-in check that detects if the index already
|
||||||
|
exists. In the event a match is found, index creation is skipped.
|
||||||
|
Without an explicit name argument, Rails can return a false positive
|
||||||
|
for `index_exists?`, causing a required index to not be created
|
||||||
|
properly. By always requiring a name for certain types of indexes, the
|
||||||
|
chance of error is greatly reduced.
|
||||||
|
|
||||||
|
## Temporary indexes
|
||||||
|
|
||||||
|
There may be times when an index is only needed temporarily.
|
||||||
|
|
||||||
|
For example, in a migration, a column of a table might be conditionally
|
||||||
|
updated. To query which columns must be updated in the
|
||||||
|
[query performance guidelines](../query_performance.md), an index is needed
|
||||||
|
that would otherwise not be used.
|
||||||
|
|
||||||
|
In these cases, consider a temporary index. To specify a
|
||||||
|
temporary index:
|
||||||
|
|
||||||
|
1. Prefix the index name with `tmp_` and follow the [naming conventions](constraint_naming_convention.md).
|
||||||
|
1. Create a follow-up issue to remove the index in the next (or future) milestone.
|
||||||
|
1. Add a comment in the migration mentioning the removal issue.
|
||||||
|
|
||||||
|
A temporary migration would look like:
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled'
|
||||||
|
|
||||||
|
def up
|
||||||
|
# Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234
|
||||||
|
add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
remove_concurrent_index_by_name :projects, INDEX_NAME
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
## Create indexes asynchronously
|
||||||
|
|
||||||
|
For very large tables, index creation can be a challenge to manage.
|
||||||
|
While `add_concurrent_index` creates indexes in a way that does not block
|
||||||
|
normal traffic, it can still be problematic when index creation runs for
|
||||||
|
many hours. Necessary database operations like `autovacuum` cannot run, and
|
||||||
|
on GitLab.com, the deployment process is blocked waiting for index
|
||||||
|
creation to finish.
|
||||||
|
|
||||||
|
To limit impact on GitLab.com, a process exists to create indexes
|
||||||
|
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||||
|
index creation can proceed at a lower level of risk.
|
||||||
|
|
||||||
|
### Schedule index creation for a low-impact time
|
||||||
|
|
||||||
|
1. [Schedule the index to be created](#schedule-the-index-to-be-created).
|
||||||
|
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||||
|
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||||
|
|
||||||
|
### Schedule the index to be created
|
||||||
|
|
||||||
|
Create an MR with a post-deployment migration which prepares the index
|
||||||
|
for asynchronous creation. An example of creating an index using
|
||||||
|
the asynchronous index helpers can be seen in the block below. This migration
|
||||||
|
enters the index name and definition into the `postgres_async_indexes`
|
||||||
|
table. The process that runs on weekends pulls indexes from this
|
||||||
|
table and attempt to create them.
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
# in db/post_migrate/
|
||||||
|
|
||||||
|
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||||
|
|
||||||
|
def up
|
||||||
|
prepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### Verify the MR was deployed and the index exists in production
|
||||||
|
|
||||||
|
You can verify if the post-deploy migration was executed on GitLab.com by:
|
||||||
|
|
||||||
|
- Executing `/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`,
|
||||||
|
the post-deploy migration has been executed in the production database. More details in this
|
||||||
|
[guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom).
|
||||||
|
- Use a meta-command in #database-lab, such as: `\d <index_name>`.
|
||||||
|
- Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||||
|
- Ask someone in #database to check if the index exists.
|
||||||
|
- With proper access, you can also verify directly on production or in a
|
||||||
|
production clone.
|
||||||
|
|
||||||
|
### Add a migration to create the index synchronously
|
||||||
|
|
||||||
|
After the index is verified to exist on the production database, create a second
|
||||||
|
merge request that adds the index synchronously. The schema changes must be
|
||||||
|
updated and committed to `structure.sql` in this second merge request.
|
||||||
|
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||||
|
migration as expected for other installations. The below block
|
||||||
|
demonstrates how to create the second migration for the previous
|
||||||
|
asynchronous example.
|
||||||
|
|
||||||
|
**WARNING:**
|
||||||
|
Verify that the index exists in production before merging a second migration with `add_concurrent_index`.
|
||||||
|
If the second migration is deployed before the index has been created,
|
||||||
|
the index is created synchronously when the second migration executes.
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
# in db/post_migrate/
|
||||||
|
|
||||||
|
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||||
|
|
||||||
|
disable_ddl_transaction!
|
||||||
|
|
||||||
|
def up
|
||||||
|
add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test database index changes locally
|
||||||
|
|
||||||
|
You must test the database index changes locally before creating a merge request.
|
||||||
|
|
||||||
|
### Verify indexes created asynchronously
|
||||||
|
|
||||||
|
Use the asynchronous index helpers on your local environment to test changes for creating an index:
|
||||||
|
|
||||||
|
1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||||
|
1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table.
|
||||||
|
1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously.
|
||||||
|
1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists.
|
||||||
|
|
||||||
|
## Drop indexes asynchronously
|
||||||
|
|
||||||
|
For very large tables, index destruction can be a challenge to manage.
|
||||||
|
While `remove_concurrent_index` removes indexes in a way that does not block
|
||||||
|
normal traffic, it can still be problematic if index destruction runs for
|
||||||
|
during autovacuum. Necessary database operations like `autovacuum` cannot run, and
|
||||||
|
the deployment process on GitLab.com is blocked while waiting for index
|
||||||
|
destruction to finish.
|
||||||
|
|
||||||
|
To limit the impact on GitLab.com, use the following process to remove indexes
|
||||||
|
asynchronously during weekend hours. Due to generally lower traffic and fewer deployments,
|
||||||
|
index destruction can proceed at a lower level of risk.
|
||||||
|
|
||||||
|
1. [Schedule the index to be removed](#schedule-the-index-to-be-removed).
|
||||||
|
1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
|
||||||
|
1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
|
||||||
|
|
||||||
|
### Schedule the index to be removed
|
||||||
|
|
||||||
|
Create an MR with a post-deployment migration which prepares the index
|
||||||
|
for asynchronous destruction. For example. to destroy an index using
|
||||||
|
the asynchronous index helpers:
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
# in db/post_migrate/
|
||||||
|
|
||||||
|
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||||
|
|
||||||
|
def up
|
||||||
|
prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
This migration enters the index name and definition into the `postgres_async_indexes`
|
||||||
|
table. The process that runs on weekends pulls indexes from this table and attempt
|
||||||
|
to remove them.
|
||||||
|
|
||||||
|
You must test the database index changes locally before creating a merge request.
|
||||||
|
|
||||||
|
### Verify the MR was deployed and the index exists in production
|
||||||
|
|
||||||
|
You can verify if the MR was deployed to GitLab.com with
|
||||||
|
`/chatops run auto_deploy status <merge_sha>`. To verify the existence of
|
||||||
|
the index, you can:
|
||||||
|
|
||||||
|
- Use a meta-command in `#database-lab`, for example: `\d <index_name>`.
|
||||||
|
- Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID).
|
||||||
|
- Ask someone in `#database` to check if the index exists.
|
||||||
|
- If you have access, you can verify directly on production or in a
|
||||||
|
production clone.
|
||||||
|
|
||||||
|
### Add a migration to destroy the index synchronously
|
||||||
|
|
||||||
|
After you verify the index exists in the production database, create a second
|
||||||
|
merge request that removes the index synchronously. The schema changes must be
|
||||||
|
updated and committed to `structure.sql` in this second merge request.
|
||||||
|
The synchronous migration results in a no-op on GitLab.com, but you should still add the
|
||||||
|
migration as expected for other installations. For example, to
|
||||||
|
create the second migration for the previous asynchronous example:
|
||||||
|
|
||||||
|
**WARNING:**
|
||||||
|
Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`.
|
||||||
|
If the second migration is deployed before the index has been destroyed,
|
||||||
|
the index is destroyed synchronously when the second migration executes.
|
||||||
|
|
||||||
|
```ruby
|
||||||
|
# in db/post_migrate/
|
||||||
|
|
||||||
|
INDEX_NAME = 'index_ci_builds_on_some_column'
|
||||||
|
|
||||||
|
disable_ddl_transaction!
|
||||||
|
|
||||||
|
def up
|
||||||
|
remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
add_concurrent_index :ci_builds, :some_column, INDEX_NAME
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### Verify indexes removed asynchronously
|
||||||
|
|
||||||
|
To test changes for removing an index, use the asynchronous index helpers on your local environment:
|
||||||
|
|
||||||
|
1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console.
|
||||||
|
1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table.
|
||||||
|
1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously.
|
||||||
|
1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md)
|
||||||
|
command `gdk psql` and run `\d <index_name>` to check that the destroyed index no longer exists.
|
|
@ -0,0 +1,62 @@
|
||||||
|
---
|
||||||
|
stage: Data Stores
|
||||||
|
group: Database
|
||||||
|
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||||
|
---
|
||||||
|
|
||||||
|
# Database query comments with Marginalia
|
||||||
|
|
||||||
|
The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add
|
||||||
|
query comments containing application related context information to PostgreSQL
|
||||||
|
queries generated by ActiveRecord.
|
||||||
|
|
||||||
|
It is very useful for tracing problematic queries back to the application source.
|
||||||
|
|
||||||
|
An engineer during an on-call incident has the full context of a query
|
||||||
|
and its application source from the comments.
|
||||||
|
|
||||||
|
## Metadata information in comments
|
||||||
|
|
||||||
|
Queries generated from **Rails** include the following metadata in comments:
|
||||||
|
|
||||||
|
- `application`
|
||||||
|
- `correlation_id`
|
||||||
|
- `endpoint_id`
|
||||||
|
- `line`
|
||||||
|
|
||||||
|
Queries generated from **Sidekiq** workers include the following metadata
|
||||||
|
in comments:
|
||||||
|
|
||||||
|
- `application`
|
||||||
|
- `jid`
|
||||||
|
- `correlation_id`
|
||||||
|
- `endpoint_id`
|
||||||
|
- `line`
|
||||||
|
|
||||||
|
`endpoint_id` is a single field that can represent any endpoint in the application:
|
||||||
|
|
||||||
|
- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`.
|
||||||
|
- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`.
|
||||||
|
- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`.
|
||||||
|
|
||||||
|
`line` is not present in production logs due to the additional overhead required.
|
||||||
|
|
||||||
|
Examples of queries with comments:
|
||||||
|
|
||||||
|
- Rails:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
/*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1
|
||||||
|
```
|
||||||
|
|
||||||
|
- Grape:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
/*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1
|
||||||
|
```
|
||||||
|
|
||||||
|
- Sidekiq:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
/*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3
|
||||||
|
```
|
|
@ -36,17 +36,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w
|
||||||
|
|
||||||
## Debugging
|
## Debugging
|
||||||
|
|
||||||
- Tracing the source of an SQL query using query comments with [Marginalia](../database_query_comments.md)
|
- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md)
|
||||||
- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs)
|
- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs)
|
||||||
|
|
||||||
## Best practices
|
## Best practices
|
||||||
|
|
||||||
- [Adding database indexes](../adding_database_indexes.md)
|
- [Adding database indexes](adding_database_indexes.md)
|
||||||
- [Foreign keys & associations](../foreign_keys.md)
|
- [Foreign keys & associations](../foreign_keys.md)
|
||||||
- [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md)
|
- [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md)
|
||||||
- [`NOT NULL` constraints](not_null_constraints.md)
|
- [`NOT NULL` constraints](not_null_constraints.md)
|
||||||
- [Strings and the Text data type](strings_and_the_text_data_type.md)
|
- [Strings and the Text data type](strings_and_the_text_data_type.md)
|
||||||
- [Single table inheritance](../single_table_inheritance.md)
|
- [Single table inheritance](single_table_inheritance.md)
|
||||||
- [Polymorphic associations](polymorphic_associations.md)
|
- [Polymorphic associations](polymorphic_associations.md)
|
||||||
- [Serializing data](../serializing_data.md)
|
- [Serializing data](../serializing_data.md)
|
||||||
- [Hash indexes](../hash_indexes.md)
|
- [Hash indexes](../hash_indexes.md)
|
||||||
|
|
|
@ -1,62 +1,11 @@
|
||||||
---
|
---
|
||||||
stage: Data Stores
|
redirect_to: 'database/database_query_comments.md'
|
||||||
group: Database
|
remove_date: '2022-11-05'
|
||||||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# Database query comments with Marginalia
|
This document was moved to [another location](database/database_query_comments.md).
|
||||||
|
|
||||||
The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add
|
<!-- This redirect file can be deleted after <2022-11-05>. -->
|
||||||
query comments containing application related context information to PostgreSQL
|
<!-- Redirects that point to other docs in the same project expire in three months. -->
|
||||||
queries generated by ActiveRecord.
|
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
|
||||||
|
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
|
||||||
It is very useful for tracing problematic queries back to the application source.
|
|
||||||
|
|
||||||
An engineer during an on-call incident has the full context of a query
|
|
||||||
and its application source from the comments.
|
|
||||||
|
|
||||||
## Metadata information in comments
|
|
||||||
|
|
||||||
Queries generated from **Rails** include the following metadata in comments:
|
|
||||||
|
|
||||||
- `application`
|
|
||||||
- `correlation_id`
|
|
||||||
- `endpoint_id`
|
|
||||||
- `line`
|
|
||||||
|
|
||||||
Queries generated from **Sidekiq** workers include the following metadata
|
|
||||||
in comments:
|
|
||||||
|
|
||||||
- `application`
|
|
||||||
- `jid`
|
|
||||||
- `correlation_id`
|
|
||||||
- `endpoint_id`
|
|
||||||
- `line`
|
|
||||||
|
|
||||||
`endpoint_id` is a single field that can represent any endpoint in the application:
|
|
||||||
|
|
||||||
- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`.
|
|
||||||
- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`.
|
|
||||||
- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`.
|
|
||||||
|
|
||||||
`line` is not present in production logs due to the additional overhead required.
|
|
||||||
|
|
||||||
Examples of queries with comments:
|
|
||||||
|
|
||||||
- Rails:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
/*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1
|
|
||||||
```
|
|
||||||
|
|
||||||
- Grape:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
/*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1
|
|
||||||
```
|
|
||||||
|
|
||||||
- Sidekiq:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
/*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3
|
|
||||||
```
|
|
||||||
|
|
|
@ -120,7 +120,7 @@ columns manually for existing tables as this causes confusion to
|
||||||
other people using `db/structure.sql` generated by Rails.
|
other people using `db/structure.sql` generated by Rails.
|
||||||
|
|
||||||
NOTE:
|
NOTE:
|
||||||
[Creating an index asynchronously requires two merge requests.](adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously)
|
[Creating an index asynchronously requires two merge requests.](database/adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously)
|
||||||
When done, commit the schema change in the merge request
|
When done, commit the schema change in the merge request
|
||||||
that adds the index with `add_concurrent_index`.
|
that adds the index with `add_concurrent_index`.
|
||||||
|
|
||||||
|
@ -612,7 +612,7 @@ might not be required, like:
|
||||||
|
|
||||||
Additionally, wide indexes are not required to match all filter criteria of queries, we just need
|
Additionally, wide indexes are not required to match all filter criteria of queries, we just need
|
||||||
to cover enough columns so that the index lookup has a small enough selectivity. Please review our
|
to cover enough columns so that the index lookup has a small enough selectivity. Please review our
|
||||||
[Adding Database indexes](adding_database_indexes.md) guide for more details.
|
[Adding Database indexes](database/adding_database_indexes.md) guide for more details.
|
||||||
|
|
||||||
When adding an index to a non-empty table make sure to use the method
|
When adding an index to a non-empty table make sure to use the method
|
||||||
`add_concurrent_index` instead of the regular `add_index` method.
|
`add_concurrent_index` instead of the regular `add_index` method.
|
||||||
|
@ -641,7 +641,7 @@ end
|
||||||
|
|
||||||
You must explicitly name indexes that are created with more complex
|
You must explicitly name indexes that are created with more complex
|
||||||
definitions beyond table name, column names, and uniqueness constraint.
|
definitions beyond table name, column names, and uniqueness constraint.
|
||||||
Consult the [Adding Database Indexes](adding_database_indexes.md#requirements-for-naming-indexes)
|
Consult the [Adding Database Indexes](database/adding_database_indexes.md#requirements-for-naming-indexes)
|
||||||
guide for more details.
|
guide for more details.
|
||||||
|
|
||||||
If you need to add a unique index, please keep in mind there is the possibility
|
If you need to add a unique index, please keep in mind there is the possibility
|
||||||
|
@ -659,7 +659,7 @@ If a migration requires conditional logic based on the absence or
|
||||||
presence of an index, you must test for existence of that index using
|
presence of an index, you must test for existence of that index using
|
||||||
its name. This helps avoids problems with how Rails compares index definitions,
|
its name. This helps avoids problems with how Rails compares index definitions,
|
||||||
which can lead to unexpected results. For more details, review the
|
which can lead to unexpected results. For more details, review the
|
||||||
[Adding Database Indexes](adding_database_indexes.md#why-explicit-names-are-required)
|
[Adding Database Indexes](database/adding_database_indexes.md#why-explicit-names-are-required)
|
||||||
guide.
|
guide.
|
||||||
|
|
||||||
The easiest way to test for existence of an index by name is to use the
|
The easiest way to test for existence of an index by name is to use the
|
||||||
|
@ -1198,7 +1198,7 @@ If using a model in the migrations, you should first
|
||||||
using `reset_column_information`.
|
using `reset_column_information`.
|
||||||
|
|
||||||
If using a model that leverages single table inheritance (STI), there are [special
|
If using a model that leverages single table inheritance (STI), there are [special
|
||||||
considerations](single_table_inheritance.md#in-migrations).
|
considerations](database/single_table_inheritance.md#in-migrations).
|
||||||
|
|
||||||
This avoids problems where a column that you are using was altered and cached
|
This avoids problems where a column that you are using was altered and cached
|
||||||
in a previous migration.
|
in a previous migration.
|
||||||
|
|
|
@ -32,12 +32,10 @@ When you create a release, or after, you can:
|
||||||
- Add release notes.
|
- Add release notes.
|
||||||
- Add a message for the Git tag associated with the release.
|
- Add a message for the Git tag associated with the release.
|
||||||
- [Associate milestones with it](#associate-milestones-with-a-release).
|
- [Associate milestones with it](#associate-milestones-with-a-release).
|
||||||
- Attach [release assets](#release-assets), like runbooks or packages.
|
- Attach [release assets](release_fields.md#release-assets), like runbooks or packages.
|
||||||
|
|
||||||
## View releases
|
## View releases
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8.
|
|
||||||
|
|
||||||
To view a list of releases:
|
To view a list of releases:
|
||||||
|
|
||||||
- On the left sidebar, select **Deployments > Releases**, or
|
- On the left sidebar, select **Deployments > Releases**, or
|
||||||
|
@ -81,18 +79,18 @@ To create a release in the Releases page:
|
||||||
|
|
||||||
1. On the top bar, select **Menu > Projects** and find your project.
|
1. On the top bar, select **Menu > Projects** and find your project.
|
||||||
1. On the left sidebar, select **Deployments > Releases** and select **New release**.
|
1. On the left sidebar, select **Deployments > Releases** and select **New release**.
|
||||||
1. From the [**Tag name**](#tag-name) dropdown, either:
|
1. From the [**Tag name**](release_fields.md#tag-name) dropdown, either:
|
||||||
- Select an existing Git tag. Selecting an existing tag that is already associated with a release
|
- Select an existing Git tag. Selecting an existing tag that is already associated with a release
|
||||||
results in a validation error.
|
results in a validation error.
|
||||||
- Enter a new Git tag name.
|
- Enter a new Git tag name.
|
||||||
1. From the **Create from** dropdown, select a branch or commit SHA to use when creating the
|
1. From the **Create from** dropdown, select a branch or commit SHA to use when creating the
|
||||||
new tag.
|
new tag.
|
||||||
1. Optional. Enter additional information about the release, including:
|
1. Optional. Enter additional information about the release, including:
|
||||||
- [Title](#title).
|
- [Title](release_fields.md#title).
|
||||||
- [Milestones](#associate-milestones-with-a-release).
|
- [Milestones](#associate-milestones-with-a-release).
|
||||||
- [Release notes](#release-notes-description).
|
- [Release notes](release_fields.md#release-notes-description).
|
||||||
- Whether or not to include the [Tag message](../../../topics/git/tags.md).
|
- Whether or not to include the [Tag message](../../../topics/git/tags.md).
|
||||||
- [Asset links](#links).
|
- [Asset links](release_fields.md#links).
|
||||||
1. Select **Create release**.
|
1. Select **Create release**.
|
||||||
|
|
||||||
### Create a release in the Tags page
|
### Create a release in the Tags page
|
||||||
|
@ -298,9 +296,6 @@ is not available.
|
||||||
|
|
||||||
## Edit a release
|
## Edit a release
|
||||||
|
|
||||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6.
|
|
||||||
> - Asset link editing [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10.
|
|
||||||
|
|
||||||
Only users with at least the Developer role can edit releases.
|
Only users with at least the Developer role can edit releases.
|
||||||
Read more about [Release permissions](#release-permissions).
|
Read more about [Release permissions](#release-permissions).
|
||||||
|
|
||||||
|
@ -428,277 +423,6 @@ complete overlapping period.
|
||||||
|
|
||||||
For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md).
|
For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md).
|
||||||
|
|
||||||
## Release fields
|
|
||||||
|
|
||||||
The following fields are available when you create or edit a release.
|
|
||||||
|
|
||||||
### Title
|
|
||||||
|
|
||||||
The release title can be customized using the **Release title** field when
|
|
||||||
creating or editing a release. If no title is provided, the release's tag name
|
|
||||||
is used instead.
|
|
||||||
|
|
||||||
### Tag name
|
|
||||||
|
|
||||||
The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/)
|
|
||||||
for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the
|
|
||||||
[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning).
|
|
||||||
|
|
||||||
For example, for GitLab version `10.5.7`:
|
|
||||||
|
|
||||||
- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`.
|
|
||||||
- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`.
|
|
||||||
- `7` represents the patch number.
|
|
||||||
|
|
||||||
Any part of the version number can be multiple digits, for example, `13.10.11`.
|
|
||||||
|
|
||||||
### Release notes description
|
|
||||||
|
|
||||||
Every release has a description. You can add any text you like, but we recommend
|
|
||||||
including a changelog to describe the content of your release. This helps users
|
|
||||||
quickly scan the differences between each release you publish.
|
|
||||||
|
|
||||||
[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can
|
|
||||||
be included in Release note descriptions by selecting **Include tag message in
|
|
||||||
the release notes**.
|
|
||||||
|
|
||||||
Description supports [Markdown](../../markdown.md).
|
|
||||||
|
|
||||||
### Release assets
|
|
||||||
|
|
||||||
A release contains the following types of assets:
|
|
||||||
|
|
||||||
- [Source code](#source-code)
|
|
||||||
- [Link](#links)
|
|
||||||
|
|
||||||
#### Source code
|
|
||||||
|
|
||||||
GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar`
|
|
||||||
archived source code from the given Git tag. These are read-only assets.
|
|
||||||
|
|
||||||
#### Links
|
|
||||||
|
|
||||||
A link is any URL which can point to whatever you like: documentation, built
|
|
||||||
binaries, or other related materials. These can be both internal or external
|
|
||||||
links from your GitLab instance.
|
|
||||||
Each link as an asset has the following attributes:
|
|
||||||
|
|
||||||
| Attribute | Description | Required |
|
|
||||||
| ---- | ----------- | --- |
|
|
||||||
| `name` | The name of the link. | Yes |
|
|
||||||
| `url` | The URL to download a file. | Yes |
|
|
||||||
| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No |
|
|
||||||
| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No |
|
|
||||||
|
|
||||||
##### Permanent link to latest release
|
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
|
||||||
|
|
||||||
Latest release page is accessible through a permanent URL.
|
|
||||||
GitLab will redirect to the latest release page URL when it is visited.
|
|
||||||
|
|
||||||
The format of the URL is:
|
|
||||||
|
|
||||||
```plaintext
|
|
||||||
https://host/namespace/project/-/releases/permalink/latest
|
|
||||||
```
|
|
||||||
|
|
||||||
We also support, suffix path carry forward on the redirect to the latest release.
|
|
||||||
Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`.
|
|
||||||
|
|
||||||
Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage.
|
|
||||||
|
|
||||||
###### Sorting preferences
|
|
||||||
|
|
||||||
By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945).
|
|
||||||
|
|
||||||
##### Permanent links to release assets
|
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9.
|
|
||||||
|
|
||||||
The assets associated with a release are accessible through a permanent URL.
|
|
||||||
GitLab always redirects this URL to the actual asset
|
|
||||||
location, so even if the assets move to a different location, you can continue
|
|
||||||
to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute.
|
|
||||||
|
|
||||||
The format of the URL is:
|
|
||||||
|
|
||||||
```plaintext
|
|
||||||
https://host/namespace/project/-/releases/:release/downloads/:filepath
|
|
||||||
```
|
|
||||||
|
|
||||||
If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org`
|
|
||||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "linux amd64",
|
|
||||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
|
||||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
|
||||||
"link_type": "other"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
This asset has a direct link of:
|
|
||||||
|
|
||||||
```plaintext
|
|
||||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64
|
|
||||||
```
|
|
||||||
|
|
||||||
The physical location of the asset can change at any time and the direct link remains unchanged.
|
|
||||||
|
|
||||||
##### Permanent links to latest release assets
|
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
|
||||||
|
|
||||||
The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*.
|
|
||||||
|
|
||||||
The format of the URL is:
|
|
||||||
|
|
||||||
```plaintext
|
|
||||||
https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath
|
|
||||||
```
|
|
||||||
|
|
||||||
If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org`
|
|
||||||
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "linux amd64",
|
|
||||||
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
|
||||||
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
|
||||||
"link_type": "other"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
This asset has a direct link of:
|
|
||||||
|
|
||||||
```plaintext
|
|
||||||
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64
|
|
||||||
```
|
|
||||||
|
|
||||||
##### Link Types
|
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1.
|
|
||||||
|
|
||||||
The four types of links are "Runbook," "Package," "Image," and "Other."
|
|
||||||
The `link_type` parameter accepts one of the following four values:
|
|
||||||
|
|
||||||
- `runbook`
|
|
||||||
- `package`
|
|
||||||
- `image`
|
|
||||||
- `other` (default)
|
|
||||||
|
|
||||||
This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
|
|
||||||
|
|
||||||
##### Use a generic package for attaching binaries
|
|
||||||
|
|
||||||
You can use [generic packages](../../packages/generic_packages/index.md)
|
|
||||||
to store any artifacts from a release or tag pipeline,
|
|
||||||
that can also be used for attaching binary files to an individual release entry.
|
|
||||||
You basically need to:
|
|
||||||
|
|
||||||
1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file).
|
|
||||||
1. [Attach the package link to the release](#links).
|
|
||||||
|
|
||||||
The following example generates release assets, publishes them
|
|
||||||
as a generic package, and then creates a release:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
stages:
|
|
||||||
- build
|
|
||||||
- upload
|
|
||||||
- release
|
|
||||||
|
|
||||||
variables:
|
|
||||||
# Package version can only contain numbers (0-9), and dots (.).
|
|
||||||
# Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion.
|
|
||||||
# See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file
|
|
||||||
PACKAGE_VERSION: "1.2.3"
|
|
||||||
DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}"
|
|
||||||
LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}"
|
|
||||||
PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}"
|
|
||||||
|
|
||||||
build:
|
|
||||||
stage: build
|
|
||||||
image: alpine:latest
|
|
||||||
rules:
|
|
||||||
- if: $CI_COMMIT_TAG
|
|
||||||
script:
|
|
||||||
- mkdir bin
|
|
||||||
- echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY}
|
|
||||||
- echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY}
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- bin/
|
|
||||||
|
|
||||||
upload:
|
|
||||||
stage: upload
|
|
||||||
image: curlimages/curl:latest
|
|
||||||
rules:
|
|
||||||
- if: $CI_COMMIT_TAG
|
|
||||||
script:
|
|
||||||
- |
|
|
||||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}"
|
|
||||||
- |
|
|
||||||
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}"
|
|
||||||
|
|
||||||
release:
|
|
||||||
# Caution, as of 2021-02-02 these assets links require a login, see:
|
|
||||||
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
|
|
||||||
stage: release
|
|
||||||
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
|
||||||
rules:
|
|
||||||
- if: $CI_COMMIT_TAG
|
|
||||||
script:
|
|
||||||
- |
|
|
||||||
release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
|
|
||||||
--assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \
|
|
||||||
--assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}"
|
|
||||||
```
|
|
||||||
|
|
||||||
PowerShell users may need to escape the double quote `"` inside a JSON
|
|
||||||
string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json`
|
|
||||||
before passing on to the `release-cli`.
|
|
||||||
For example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
release:
|
|
||||||
script:
|
|
||||||
- $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}"
|
|
||||||
- $env:assetjson = $env:asset | ConvertTo-Json
|
|
||||||
- release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson
|
|
||||||
```
|
|
||||||
|
|
||||||
NOTE:
|
|
||||||
Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md)
|
|
||||||
links to a release is not recommended, because artifacts are ephemeral and
|
|
||||||
are used to pass data in the same pipeline. This means there's a risk that
|
|
||||||
they could either expire or someone might manually delete them.
|
|
||||||
|
|
||||||
#### Number of new and total features **(FREE SAAS)**
|
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5.
|
|
||||||
|
|
||||||
On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project.
|
|
||||||
|
|
||||||
![Feature count](img/feature_count_v14_6.png "Number of features in a release")
|
|
||||||
|
|
||||||
The totals are displayed on [shields](https://shields.io/) and are generated per release by
|
|
||||||
[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake).
|
|
||||||
|
|
||||||
| Item | Formula |
|
|
||||||
| ------ | ------ |
|
|
||||||
| `New features` | Total count of release posts across all tiers for a single release in the project. |
|
|
||||||
| `Total features` | Total count of release posts in reverse order for all releases in the project. |
|
|
||||||
|
|
||||||
The counts are also shown by license tier.
|
|
||||||
|
|
||||||
| Item | Formula |
|
|
||||||
| ------ | ------ |
|
|
||||||
| `New features` | Total count of release posts across a single tier for a single release in the project. |
|
|
||||||
| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. |
|
|
||||||
|
|
||||||
## Release evidence
|
## Release evidence
|
||||||
|
|
||||||
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6.
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6.
|
||||||
|
@ -847,7 +571,7 @@ In the API:
|
||||||
- Users with the Guest role
|
- Users with the Guest role
|
||||||
have read and download access to the project releases.
|
have read and download access to the project releases.
|
||||||
This includes associated Git-tag-names, release description, author information of the releases.
|
This includes associated Git-tag-names, release description, author information of the releases.
|
||||||
However, other repository-related information, such as [source code](#source-code), [release evidence](#release-evidence) are redacted.
|
However, other repository-related information, such as [source code](release_fields.md#source-code), [release evidence](#release-evidence) are redacted.
|
||||||
|
|
||||||
### Create, update, and delete a release and its assets
|
### Create, update, and delete a release and its assets
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,274 @@
|
||||||
|
---
|
||||||
|
stage: Release
|
||||||
|
group: Release
|
||||||
|
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||||
|
---
|
||||||
|
|
||||||
|
# Release fields
|
||||||
|
|
||||||
|
The following fields are available when you create or edit a release.
|
||||||
|
|
||||||
|
## Title
|
||||||
|
|
||||||
|
The release title can be customized using the **Release title** field when
|
||||||
|
creating or editing a release. If no title is provided, the release's tag name
|
||||||
|
is used instead.
|
||||||
|
|
||||||
|
## Tag name
|
||||||
|
|
||||||
|
The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/)
|
||||||
|
for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the
|
||||||
|
[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning).
|
||||||
|
|
||||||
|
For example, for GitLab version `10.5.7`:
|
||||||
|
|
||||||
|
- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`.
|
||||||
|
- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`.
|
||||||
|
- `7` represents the patch number.
|
||||||
|
|
||||||
|
Any part of the version number can be multiple digits, for example, `13.10.11`.
|
||||||
|
|
||||||
|
## Release notes description
|
||||||
|
|
||||||
|
Every release has a description. You can add any text you like, but we recommend
|
||||||
|
including a changelog to describe the content of your release. This helps users
|
||||||
|
quickly scan the differences between each release you publish.
|
||||||
|
|
||||||
|
[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can
|
||||||
|
be included in Release note descriptions by selecting **Include tag message in
|
||||||
|
the release notes**.
|
||||||
|
|
||||||
|
Description supports [Markdown](../../markdown.md).
|
||||||
|
|
||||||
|
## Release assets
|
||||||
|
|
||||||
|
A release contains the following types of assets:
|
||||||
|
|
||||||
|
- [Source code](#source-code)
|
||||||
|
- [Link](#links)
|
||||||
|
|
||||||
|
### Source code
|
||||||
|
|
||||||
|
GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar`
|
||||||
|
archived source code from the given Git tag. These are read-only assets.
|
||||||
|
|
||||||
|
### Links
|
||||||
|
|
||||||
|
A link is any URL which can point to whatever you like: documentation, built
|
||||||
|
binaries, or other related materials. These can be both internal or external
|
||||||
|
links from your GitLab instance.
|
||||||
|
Each link as an asset has the following attributes:
|
||||||
|
|
||||||
|
| Attribute | Description | Required |
|
||||||
|
|-------------|--------------------------------------------------------------------------------------------------------------|----------|
|
||||||
|
| `name` | The name of the link. | Yes |
|
||||||
|
| `url` | The URL to download a file. | Yes |
|
||||||
|
| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No |
|
||||||
|
| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No |
|
||||||
|
|
||||||
|
#### Permanent link to latest release
|
||||||
|
|
||||||
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||||
|
|
||||||
|
Latest release page is accessible through a permanent URL.
|
||||||
|
GitLab redirects to the latest release page URL when it is visited.
|
||||||
|
|
||||||
|
The format of the URL is:
|
||||||
|
|
||||||
|
```plaintext
|
||||||
|
https://host/namespace/project/-/releases/permalink/latest
|
||||||
|
```
|
||||||
|
|
||||||
|
We also support, suffix path carry forward on the redirect to the latest release.
|
||||||
|
Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`.
|
||||||
|
|
||||||
|
Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage.
|
||||||
|
|
||||||
|
##### Sorting preferences
|
||||||
|
|
||||||
|
By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945).
|
||||||
|
|
||||||
|
#### Permanent links to release assets
|
||||||
|
|
||||||
|
The assets associated with a release are accessible through a permanent URL.
|
||||||
|
GitLab always redirects this URL to the actual asset
|
||||||
|
location, so even if the assets move to a different location, you can continue
|
||||||
|
to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute.
|
||||||
|
|
||||||
|
The format of the URL is:
|
||||||
|
|
||||||
|
```plaintext
|
||||||
|
https://host/namespace/project/-/releases/:release/downloads/:filepath
|
||||||
|
```
|
||||||
|
|
||||||
|
If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org`
|
||||||
|
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "linux amd64",
|
||||||
|
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||||
|
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||||
|
"link_type": "other"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This asset has a direct link of:
|
||||||
|
|
||||||
|
```plaintext
|
||||||
|
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64
|
||||||
|
```
|
||||||
|
|
||||||
|
The physical location of the asset can change at any time and the direct link remains unchanged.
|
||||||
|
|
||||||
|
#### Permanent links to latest release assets
|
||||||
|
|
||||||
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9.
|
||||||
|
|
||||||
|
The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*.
|
||||||
|
|
||||||
|
The format of the URL is:
|
||||||
|
|
||||||
|
```plaintext
|
||||||
|
https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath
|
||||||
|
```
|
||||||
|
|
||||||
|
If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org`
|
||||||
|
namespace and `gitlab-runner` project on `gitlab.com`, for example:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "linux amd64",
|
||||||
|
"filepath": "/binaries/gitlab-runner-linux-amd64",
|
||||||
|
"url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64",
|
||||||
|
"link_type": "other"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This asset has a direct link of:
|
||||||
|
|
||||||
|
```plaintext
|
||||||
|
https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Link Types
|
||||||
|
|
||||||
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1.
|
||||||
|
|
||||||
|
The four types of links are "Runbook," "Package," "Image," and "Other."
|
||||||
|
The `link_type` parameter accepts one of the following four values:
|
||||||
|
|
||||||
|
- `runbook`
|
||||||
|
- `package`
|
||||||
|
- `image`
|
||||||
|
- `other` (default)
|
||||||
|
|
||||||
|
This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project.
|
||||||
|
|
||||||
|
#### Use a generic package for attaching binaries
|
||||||
|
|
||||||
|
You can use [generic packages](../../packages/generic_packages/index.md)
|
||||||
|
to store any artifacts from a release or tag pipeline,
|
||||||
|
that can also be used for attaching binary files to an individual release entry.
|
||||||
|
You basically need to:
|
||||||
|
|
||||||
|
1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file).
|
||||||
|
1. [Attach the package link to the release](#links).
|
||||||
|
|
||||||
|
The following example generates release assets, publishes them
|
||||||
|
as a generic package, and then creates a release:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
stages:
|
||||||
|
- build
|
||||||
|
- upload
|
||||||
|
- release
|
||||||
|
|
||||||
|
variables:
|
||||||
|
# Package version can only contain numbers (0-9), and dots (.).
|
||||||
|
# Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion.
|
||||||
|
# See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file
|
||||||
|
PACKAGE_VERSION: "1.2.3"
|
||||||
|
DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}"
|
||||||
|
LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}"
|
||||||
|
PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}"
|
||||||
|
|
||||||
|
build:
|
||||||
|
stage: build
|
||||||
|
image: alpine:latest
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_TAG
|
||||||
|
script:
|
||||||
|
- mkdir bin
|
||||||
|
- echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY}
|
||||||
|
- echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY}
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- bin/
|
||||||
|
|
||||||
|
upload:
|
||||||
|
stage: upload
|
||||||
|
image: curlimages/curl:latest
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_TAG
|
||||||
|
script:
|
||||||
|
- |
|
||||||
|
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}"
|
||||||
|
- |
|
||||||
|
curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}"
|
||||||
|
|
||||||
|
release:
|
||||||
|
# Caution, as of 2021-02-02 these assets links require a login, see:
|
||||||
|
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
|
||||||
|
stage: release
|
||||||
|
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_TAG
|
||||||
|
script:
|
||||||
|
- |
|
||||||
|
release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
|
||||||
|
--assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \
|
||||||
|
--assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}"
|
||||||
|
```
|
||||||
|
|
||||||
|
PowerShell users may need to escape the double quote `"` inside a JSON
|
||||||
|
string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json`
|
||||||
|
before passing on to the `release-cli`.
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
release:
|
||||||
|
script:
|
||||||
|
- $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}"
|
||||||
|
- $env:assetjson = $env:asset | ConvertTo-Json
|
||||||
|
- release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson
|
||||||
|
```
|
||||||
|
|
||||||
|
NOTE:
|
||||||
|
Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md)
|
||||||
|
links to a release is not recommended, because artifacts are ephemeral and
|
||||||
|
are used to pass data in the same pipeline. This means there's a risk that
|
||||||
|
they could either expire or someone might manually delete them.
|
||||||
|
|
||||||
|
### Number of new and total features **(FREE SAAS)**
|
||||||
|
|
||||||
|
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5.
|
||||||
|
|
||||||
|
On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project.
|
||||||
|
|
||||||
|
![Feature count](img/feature_count_v14_6.png "Number of features in a release")
|
||||||
|
|
||||||
|
The totals are displayed on [shields](https://shields.io/) and are generated per release by
|
||||||
|
[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake).
|
||||||
|
|
||||||
|
| Item | Formula |
|
||||||
|
|------------------|------------------------------------------------------------------------------------|
|
||||||
|
| `New features` | Total count of release posts across all tiers for a single release in the project. |
|
||||||
|
| `Total features` | Total count of release posts in reverse order for all releases in the project. |
|
||||||
|
|
||||||
|
The counts are also shown by license tier.
|
||||||
|
|
||||||
|
| Item | Formula |
|
||||||
|
|------------------|-----------------------------------------------------------------------------------------------------|
|
||||||
|
| `New features` | Total count of release posts across a single tier for a single release in the project. |
|
||||||
|
| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. |
|
|
@ -9,7 +9,7 @@ module Gitlab
|
||||||
VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze
|
VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze
|
||||||
|
|
||||||
def self.parse(str, parse_suffix: false)
|
def self.parse(str, parse_suffix: false)
|
||||||
if str.is_a?(self.class)
|
if str.is_a?(self)
|
||||||
str
|
str
|
||||||
elsif str && m = str.match(VERSION_REGEX)
|
elsif str && m = str.match(VERSION_REGEX)
|
||||||
VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil)
|
VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil)
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
RSpec.describe 'Merge request > Context commits', :js do
|
||||||
|
let(:user) { create(:user) }
|
||||||
|
let(:project) { create(:project, :public, :repository) }
|
||||||
|
let(:merge_request) { create(:merge_request, source_project: project) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
project.add_developer(user)
|
||||||
|
|
||||||
|
sign_in(user)
|
||||||
|
|
||||||
|
visit commits_project_merge_request_path(project, merge_request)
|
||||||
|
|
||||||
|
wait_for_requests
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'opens modal' do
|
||||||
|
click_button 'Add previously merged commits'
|
||||||
|
|
||||||
|
expect(page).to have_selector('#add-review-item')
|
||||||
|
expect(page).to have_content('Add or remove previously merged commits')
|
||||||
|
end
|
||||||
|
end
|
|
@ -3,3 +3,4 @@ export * from './to_have_tracking_attributes';
|
||||||
export * from './to_match_interpolated_text';
|
export * from './to_match_interpolated_text';
|
||||||
export * from './to_validate_json_schema';
|
export * from './to_validate_json_schema';
|
||||||
export * from './to_match_expected_for_markdown';
|
export * from './to_match_expected_for_markdown';
|
||||||
|
export * from './to_equal_graphql_fixture';
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
import { isEqual } from 'lodash';
|
||||||
|
import { stripTypenames } from 'helpers/graphql_helpers';
|
||||||
|
|
||||||
|
export function toEqualGraphqlFixture(received, match) {
|
||||||
|
let clearReceived;
|
||||||
|
let clearMatch;
|
||||||
|
|
||||||
|
try {
|
||||||
|
clearReceived = JSON.parse(JSON.stringify(received));
|
||||||
|
clearMatch = stripTypenames(match);
|
||||||
|
} catch (e) {
|
||||||
|
return { message: () => 'The comparator value is not an object', pass: false };
|
||||||
|
}
|
||||||
|
const pass = isEqual(clearReceived, clearMatch);
|
||||||
|
// console.log(this.utils);
|
||||||
|
const message = pass
|
||||||
|
? () => `
|
||||||
|
Expected to not be: ${this.utils.printExpected(clearMatch)}
|
||||||
|
Received: ${this.utils.printReceived(clearReceived)}
|
||||||
|
`
|
||||||
|
: () =>
|
||||||
|
`
|
||||||
|
Expected to be: ${this.utils.printExpected(clearMatch)}
|
||||||
|
Received: ${this.utils.printReceived(clearReceived)}
|
||||||
|
`;
|
||||||
|
|
||||||
|
return { actual: received, message, pass };
|
||||||
|
}
|
|
@ -0,0 +1,58 @@
|
||||||
|
describe('custom matcher toEqualGraphqlFixture', () => {
|
||||||
|
const key = 'value';
|
||||||
|
const key2 = 'value2';
|
||||||
|
|
||||||
|
describe('positive assertion', () => {
|
||||||
|
it.each([
|
||||||
|
[{}, {}],
|
||||||
|
[{ undef: undefined }, { undef: undefined }],
|
||||||
|
[{}, { __typename: 'MyType' }],
|
||||||
|
[{ key }, { key, __typename: 'MyType' }],
|
||||||
|
[
|
||||||
|
{ obj: { key } },
|
||||||
|
{
|
||||||
|
obj: {
|
||||||
|
key,
|
||||||
|
__typename: 'MyNestedType',
|
||||||
|
},
|
||||||
|
__typename: 'MyType',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[[{ key }], [{ key }]],
|
||||||
|
[
|
||||||
|
[{ key }],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
key,
|
||||||
|
__typename: 'MyCollectionType',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
])('%j equals %j', (received, match) => {
|
||||||
|
expect(received).toEqualGraphqlFixture(match);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('negative assertion', () => {
|
||||||
|
it.each([
|
||||||
|
[{ __typename: 'MyType' }, {}],
|
||||||
|
[{ key }, { key2, __typename: 'MyType' }],
|
||||||
|
[
|
||||||
|
{ key, key2 },
|
||||||
|
{ key2, __typename: 'MyType' },
|
||||||
|
],
|
||||||
|
[[{ key }, { key2 }], [{ key }]],
|
||||||
|
[
|
||||||
|
[{ key, key2 }],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
key,
|
||||||
|
__typename: 'MyCollectionType',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
])('%j does not equal %j', (received, match) => {
|
||||||
|
expect(received).not.toEqualGraphqlFixture(match);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -214,7 +214,6 @@ Object {
|
||||||
"selfUrl": "http://localhost/releases-namespace/releases-project/-/releases/v1.1",
|
"selfUrl": "http://localhost/releases-namespace/releases-project/-/releases/v1.1",
|
||||||
},
|
},
|
||||||
"assets": Object {
|
"assets": Object {
|
||||||
"count": undefined,
|
|
||||||
"links": Array [
|
"links": Array [
|
||||||
Object {
|
Object {
|
||||||
"directAssetPath": "/binaries/awesome-app-3",
|
"directAssetPath": "/binaries/awesome-app-3",
|
||||||
|
@ -247,29 +246,22 @@ Object {
|
||||||
],
|
],
|
||||||
"sources": Array [],
|
"sources": Array [],
|
||||||
},
|
},
|
||||||
"author": undefined,
|
|
||||||
"description": "Best. Release. **Ever.** :rocket:",
|
"description": "Best. Release. **Ever.** :rocket:",
|
||||||
"evidences": Array [],
|
"evidences": Array [],
|
||||||
"milestones": Array [
|
"milestones": Array [
|
||||||
Object {
|
Object {
|
||||||
"id": "gid://gitlab/Milestone/123",
|
"id": "gid://gitlab/Milestone/123",
|
||||||
"issueStats": Object {},
|
"issueStats": Object {},
|
||||||
"stats": undefined,
|
|
||||||
"title": "12.3",
|
"title": "12.3",
|
||||||
"webPath": undefined,
|
|
||||||
"webUrl": undefined,
|
|
||||||
},
|
},
|
||||||
Object {
|
Object {
|
||||||
"id": "gid://gitlab/Milestone/124",
|
"id": "gid://gitlab/Milestone/124",
|
||||||
"issueStats": Object {},
|
"issueStats": Object {},
|
||||||
"stats": undefined,
|
|
||||||
"title": "12.4",
|
"title": "12.4",
|
||||||
"webPath": undefined,
|
|
||||||
"webUrl": undefined,
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"name": "The first release",
|
"name": "The first release",
|
||||||
"releasedAt": 2018-12-10T00:00:00.000Z,
|
"releasedAt": "2018-12-10T00:00:00.000Z",
|
||||||
"tagName": "v1.1",
|
"tagName": "v1.1",
|
||||||
"tagPath": "/releases-namespace/releases-project/-/tags/v1.1",
|
"tagPath": "/releases-namespace/releases-project/-/tags/v1.1",
|
||||||
},
|
},
|
||||||
|
|
|
@ -7,6 +7,7 @@ import {
|
||||||
convertAllReleasesGraphQLResponse,
|
convertAllReleasesGraphQLResponse,
|
||||||
convertOneReleaseGraphQLResponse,
|
convertOneReleaseGraphQLResponse,
|
||||||
} from '~/releases/util';
|
} from '~/releases/util';
|
||||||
|
import { stripTypenames } from 'helpers/graphql_helpers';
|
||||||
|
|
||||||
describe('releases/util.js', () => {
|
describe('releases/util.js', () => {
|
||||||
describe('convertGraphQLRelease', () => {
|
describe('convertGraphQLRelease', () => {
|
||||||
|
@ -136,7 +137,7 @@ describe('releases/util.js', () => {
|
||||||
describe('convertOneReleaseForEditingGraphQLResponse', () => {
|
describe('convertOneReleaseForEditingGraphQLResponse', () => {
|
||||||
it('matches snapshot', () => {
|
it('matches snapshot', () => {
|
||||||
expect(
|
expect(
|
||||||
convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse),
|
stripTypenames(convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse)),
|
||||||
).toMatchSnapshot();
|
).toMatchSnapshot();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -87,10 +87,10 @@ describe('AdminRunnerEditApp', () => {
|
||||||
await createComponentWithApollo();
|
await createComponentWithApollo();
|
||||||
|
|
||||||
expect(findRunnerUpdateForm().props()).toMatchObject({
|
expect(findRunnerUpdateForm().props()).toMatchObject({
|
||||||
runner: mockRunner,
|
|
||||||
loading: false,
|
loading: false,
|
||||||
runnerPath: mockRunnerPath,
|
runnerPath: mockRunnerPath,
|
||||||
});
|
});
|
||||||
|
expect(findRunnerUpdateForm().props('runner')).toEqualGraphqlFixture(mockRunner);
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('When there is an error', () => {
|
describe('When there is an error', () => {
|
||||||
|
|
|
@ -256,7 +256,7 @@ describe('AdminRunnerShowApp', () => {
|
||||||
await createComponent({ stubs });
|
await createComponent({ stubs });
|
||||||
|
|
||||||
expect(findJobCountBadge().text()).toBe('3');
|
expect(findJobCountBadge().text()).toBe('3');
|
||||||
expect(findRunnersJobs().props('runner')).toEqual({ ...mockRunner, ...runner });
|
expect(findRunnersJobs().props('runner')).toEqualGraphqlFixture({ ...mockRunner, ...runner });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -167,7 +167,7 @@ describe('AdminRunnersApp', () => {
|
||||||
it('shows the runners list', async () => {
|
it('shows the runners list', async () => {
|
||||||
await createComponent();
|
await createComponent();
|
||||||
|
|
||||||
expect(findRunnerList().props('runners')).toEqual(mockRunners);
|
expect(findRunnerList().props('runners')).toEqualGraphqlFixture(mockRunners);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('runner item links to the runner admin page', async () => {
|
it('runner item links to the runner admin page', async () => {
|
||||||
|
@ -188,7 +188,7 @@ describe('AdminRunnersApp', () => {
|
||||||
const runnerActions = wrapper.find('tr [data-testid="td-actions"]').find(RunnerActionsCell);
|
const runnerActions = wrapper.find('tr [data-testid="td-actions"]').find(RunnerActionsCell);
|
||||||
const runner = mockRunners[0];
|
const runner = mockRunners[0];
|
||||||
|
|
||||||
expect(runnerActions.props()).toEqual({
|
expect(runnerActions.props()).toEqualGraphqlFixture({
|
||||||
runner,
|
runner,
|
||||||
editUrl: runner.editAdminUrl,
|
editUrl: runner.editAdminUrl,
|
||||||
});
|
});
|
||||||
|
|
|
@ -73,8 +73,7 @@ describe('RunnerJobs', () => {
|
||||||
it('Shows jobs', () => {
|
it('Shows jobs', () => {
|
||||||
const jobs = findRunnerJobsTable().props('jobs');
|
const jobs = findRunnerJobsTable().props('jobs');
|
||||||
|
|
||||||
expect(jobs).toHaveLength(mockJobs.length);
|
expect(jobs).toEqualGraphqlFixture(mockJobs);
|
||||||
expect(jobs[0]).toMatchObject(mockJobs[0]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('When "Next" page is clicked', () => {
|
describe('When "Next" page is clicked', () => {
|
||||||
|
|
|
@ -107,7 +107,7 @@ describe('GroupRunnerShowApp', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders runner details component', () => {
|
it('renders runner details component', () => {
|
||||||
expect(findRunnerDetails().props('runner')).toEqual(mockRunner);
|
expect(findRunnerDetails().props('runner')).toEqualGraphqlFixture(mockRunner);
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when runner cannot be updated', () => {
|
describe('when runner cannot be updated', () => {
|
||||||
|
|
|
@ -167,7 +167,7 @@ describe('GroupRunnersApp', () => {
|
||||||
await createComponent();
|
await createComponent();
|
||||||
|
|
||||||
const runners = findRunnerList().props('runners');
|
const runners = findRunnerList().props('runners');
|
||||||
expect(runners).toEqual(mockGroupRunnersEdges.map(({ node }) => node));
|
expect(runners).toEqualGraphqlFixture(mockGroupRunnersEdges.map(({ node }) => node));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('requests the runners with group path and no other filters', async () => {
|
it('requests the runners with group path and no other filters', async () => {
|
||||||
|
|
|
@ -84,6 +84,7 @@ RSpec.describe Gitlab::VersionInfo do
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '.parse' do
|
describe '.parse' do
|
||||||
|
it { expect(described_class.parse(described_class.new(1, 0, 0))).to eq(@v1_0_0) }
|
||||||
it { expect(described_class.parse("1.0.0")).to eq(@v1_0_0) }
|
it { expect(described_class.parse("1.0.0")).to eq(@v1_0_0) }
|
||||||
it { expect(described_class.parse("1.0.0.1")).to eq(@v1_0_0) }
|
it { expect(described_class.parse("1.0.0.1")).to eq(@v1_0_0) }
|
||||||
it { expect(described_class.parse("1.0.0-ee")).to eq(@v1_0_0) }
|
it { expect(described_class.parse("1.0.0-ee")).to eq(@v1_0_0) }
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
require 'action_dispatch/testing/test_request'
|
require 'action_dispatch/testing/test_request'
|
||||||
require 'fileutils'
|
require 'fileutils'
|
||||||
|
require 'graphlyte'
|
||||||
|
|
||||||
require_relative '../../../lib/gitlab/popen'
|
require_relative '../../../lib/gitlab/popen'
|
||||||
|
|
||||||
|
@ -47,7 +48,8 @@ module JavaScriptFixturesHelpers
|
||||||
path = Rails.root / base / query_path
|
path = Rails.root / base / query_path
|
||||||
queries = Gitlab::Graphql::Queries.find(path)
|
queries = Gitlab::Graphql::Queries.find(path)
|
||||||
if queries.length == 1
|
if queries.length == 1
|
||||||
queries.first.text(mode: Gitlab.ee? ? :ee : :ce )
|
query = queries.first.text(mode: Gitlab.ee? ? :ee : :ce )
|
||||||
|
inflate_query_with_typenames(query)
|
||||||
else
|
else
|
||||||
raise "Could not find query file at #{path}, please check your query_path" % path
|
raise "Could not find query file at #{path}, please check your query_path" % path
|
||||||
end
|
end
|
||||||
|
@ -55,6 +57,23 @@ module JavaScriptFixturesHelpers
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
|
# Private: Parse a GraphQL query and inflate the fields with a __typename
|
||||||
|
#
|
||||||
|
# query - the GraqhQL query to parse
|
||||||
|
def inflate_query_with_typenames(query, doc: Graphlyte.parse(query))
|
||||||
|
typename_editor.edit(doc)
|
||||||
|
|
||||||
|
doc.to_s
|
||||||
|
end
|
||||||
|
|
||||||
|
def typename_editor
|
||||||
|
typename = Graphlyte::Syntax::Field.new(name: '__typename')
|
||||||
|
|
||||||
|
@editor ||= Graphlyte::Editor.new.on_field do |field|
|
||||||
|
field.selection << typename unless field.selection.empty? || field.selection.map(&:name).include?('__typename')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
# Private: Store a response object as fixture file
|
# Private: Store a response object as fixture file
|
||||||
#
|
#
|
||||||
# response - string or response object to store
|
# response - string or response object to store
|
||||||
|
|
|
@ -17,10 +17,6 @@ workflow:
|
||||||
script:
|
script:
|
||||||
- bundle exec rspec
|
- bundle exec rspec
|
||||||
|
|
||||||
rspec-2.6:
|
|
||||||
image: "ruby:2.6"
|
|
||||||
extends: .rspec
|
|
||||||
|
|
||||||
rspec-2.7:
|
rspec-2.7:
|
||||||
image: "ruby:2.7"
|
image: "ruby:2.7"
|
||||||
extends: .rspec
|
extends: .rspec
|
||||||
|
|
|
@ -17,14 +17,10 @@ workflow:
|
||||||
script:
|
script:
|
||||||
- bundle exec rspec
|
- bundle exec rspec
|
||||||
|
|
||||||
rspec-2.6:
|
|
||||||
image: "ruby:2.6"
|
|
||||||
extends: .rspec
|
|
||||||
|
|
||||||
rspec-2.7:
|
rspec-2.7:
|
||||||
image: "ruby:2.7"
|
image: "ruby:2.7"
|
||||||
extends: .rspec
|
extends: .rspec
|
||||||
|
|
||||||
rspec-3.0:
|
rspec-3.0:
|
||||||
image: "ruby:3.0"
|
image: "ruby:3.0"
|
||||||
extends: .rspec
|
extends: .rspec
|
||||||
|
|
|
@ -17,10 +17,6 @@ workflow:
|
||||||
script:
|
script:
|
||||||
- bundle exec rspec
|
- bundle exec rspec
|
||||||
|
|
||||||
rspec-2.6:
|
|
||||||
image: "ruby:2.6"
|
|
||||||
extends: .rspec
|
|
||||||
|
|
||||||
rspec-2.7:
|
rspec-2.7:
|
||||||
image: "ruby:2.7"
|
image: "ruby:2.7"
|
||||||
extends: .rspec
|
extends: .rspec
|
||||||
|
|
|
@ -9,49 +9,51 @@ PATH
|
||||||
GEM
|
GEM
|
||||||
remote: http://rubygems.org/
|
remote: http://rubygems.org/
|
||||||
specs:
|
specs:
|
||||||
activesupport (5.0.0.1)
|
activesupport (7.0.3.1)
|
||||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||||
i18n (~> 0.7)
|
i18n (>= 1.6, < 2)
|
||||||
minitest (~> 5.1)
|
minitest (>= 5.1)
|
||||||
tzinfo (~> 1.1)
|
tzinfo (~> 2.0)
|
||||||
addressable (2.5.2)
|
addressable (2.8.0)
|
||||||
public_suffix (>= 2.0.2, < 4.0)
|
public_suffix (>= 2.0.2, < 5.0)
|
||||||
concurrent-ruby (1.0.5)
|
concurrent-ruby (1.1.10)
|
||||||
crack (0.4.3)
|
crack (0.4.5)
|
||||||
safe_yaml (~> 1.0.0)
|
rexml
|
||||||
diff-lcs (1.2.5)
|
diff-lcs (1.5.0)
|
||||||
hashdiff (0.3.6)
|
hashdiff (1.0.1)
|
||||||
hashie (3.4.3)
|
hashie (5.0.0)
|
||||||
i18n (0.8.1)
|
i18n (1.12.0)
|
||||||
mini_portile2 (2.1.0)
|
concurrent-ruby (~> 1.0)
|
||||||
minitest (5.10.1)
|
mini_portile2 (2.8.0)
|
||||||
nokogiri (1.6.8.1)
|
minitest (5.16.2)
|
||||||
mini_portile2 (~> 2.1.0)
|
nokogiri (1.13.8)
|
||||||
omniauth (1.3.1)
|
mini_portile2 (~> 2.8.0)
|
||||||
hashie (>= 1.2, < 4)
|
racc (~> 1.4)
|
||||||
rack (>= 1.0, < 3)
|
omniauth (1.9.1)
|
||||||
public_suffix (3.0.0)
|
hashie (>= 3.4.6)
|
||||||
rack (1.6.4)
|
rack (>= 1.6.2, < 3)
|
||||||
rack-test (0.6.3)
|
public_suffix (4.0.7)
|
||||||
rack (>= 1.0)
|
racc (1.6.0)
|
||||||
rake (10.5.0)
|
rack (2.2.4)
|
||||||
|
rack-test (2.0.2)
|
||||||
|
rack (>= 1.3)
|
||||||
|
rake (13.0.6)
|
||||||
rexml (3.2.5)
|
rexml (3.2.5)
|
||||||
rspec (3.0.0)
|
rspec (3.11.0)
|
||||||
rspec-core (~> 3.0.0)
|
rspec-core (~> 3.11.0)
|
||||||
rspec-expectations (~> 3.0.0)
|
rspec-expectations (~> 3.11.0)
|
||||||
rspec-mocks (~> 3.0.0)
|
rspec-mocks (~> 3.11.0)
|
||||||
rspec-core (3.0.4)
|
rspec-core (3.11.0)
|
||||||
rspec-support (~> 3.0.0)
|
rspec-support (~> 3.11.0)
|
||||||
rspec-expectations (3.0.4)
|
rspec-expectations (3.11.0)
|
||||||
diff-lcs (>= 1.2.0, < 2.0)
|
diff-lcs (>= 1.2.0, < 2.0)
|
||||||
rspec-support (~> 3.0.0)
|
rspec-support (~> 3.11.0)
|
||||||
rspec-mocks (3.0.4)
|
rspec-mocks (3.11.1)
|
||||||
rspec-support (~> 3.0.0)
|
diff-lcs (>= 1.2.0, < 2.0)
|
||||||
rspec-support (3.0.4)
|
rspec-support (~> 3.11.0)
|
||||||
safe_yaml (1.0.4)
|
rspec-support (3.11.0)
|
||||||
thread_safe (0.3.6)
|
tzinfo (2.0.5)
|
||||||
tzinfo (1.2.2)
|
concurrent-ruby (~> 1.0)
|
||||||
thread_safe (~> 0.1)
|
|
||||||
webmock (3.0.1)
|
webmock (3.0.1)
|
||||||
addressable (>= 2.3.6)
|
addressable (>= 2.3.6)
|
||||||
crack (>= 0.3.2)
|
crack (>= 0.3.2)
|
||||||
|
@ -67,8 +69,8 @@ DEPENDENCIES
|
||||||
rack-test
|
rack-test
|
||||||
rake
|
rake
|
||||||
rexml (~> 3.2.5)
|
rexml (~> 3.2.5)
|
||||||
rspec (~> 3.0.0)
|
rspec (>= 3.4)
|
||||||
webmock (~> 3.0.0)
|
webmock (~> 3.0.0)
|
||||||
|
|
||||||
BUNDLED WITH
|
BUNDLED WITH
|
||||||
2.3.15
|
2.3.19
|
||||||
|
|
|
@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
||||||
gem.add_development_dependency(%q<rake>, [">= 0"])
|
gem.add_development_dependency(%q<rake>, [">= 0"])
|
||||||
gem.add_development_dependency(%q<rack-test>, [">= 0"])
|
gem.add_development_dependency(%q<rack-test>, [">= 0"])
|
||||||
gem.add_development_dependency(%q<rexml>, ["~> 3.2.5"])
|
gem.add_development_dependency(%q<rexml>, ["~> 3.2.5"])
|
||||||
gem.add_development_dependency(%q<rspec>, ["~> 3.0.0"])
|
gem.add_development_dependency(%q<rspec>, [">= 3.4"])
|
||||||
gem.add_development_dependency(%q<webmock>, ["~> 3.0.0"])
|
gem.add_development_dependency(%q<webmock>, ["~> 3.0.0"])
|
||||||
gem.add_development_dependency(%q<bundler>, ["> 1.0.0"])
|
gem.add_development_dependency(%q<bundler>, ["> 1.0.0"])
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue