diff --git a/.rubocop_todo/naming/heredoc_delimiter_naming.yml b/.rubocop_todo/naming/heredoc_delimiter_naming.yml index c1d058240a4..29276e529af 100644 --- a/.rubocop_todo/naming/heredoc_delimiter_naming.yml +++ b/.rubocop_todo/naming/heredoc_delimiter_naming.yml @@ -1,8 +1,5 @@ --- Naming/HeredocDelimiterNaming: - # Offense count: 388 - # Temporarily disabled due to too many offenses - Enabled: false Exclude: - 'app/models/ci/build_trace_chunks/redis_base.rb' - 'app/models/concerns/counter_attribute.rb' @@ -31,6 +28,7 @@ Naming/HeredocDelimiterNaming: - 'ee/spec/services/security/security_orchestration_policies/policy_commit_service_spec.rb' - 'ee/spec/support/helpers/ee/ldap_helpers.rb' - 'ee/spec/tasks/gitlab/elastic_rake_spec.rb' + - 'lib/api/metadata.rb' - 'lib/api/version.rb' - 'lib/backup/helper.rb' - 'lib/feature/shared.rb' @@ -46,14 +44,14 @@ Naming/HeredocDelimiterNaming: - 'lib/tasks/gitlab/docs/compile_deprecations.rake' - 'lib/tasks/gitlab/password.rake' - 'qa/qa/scenario/test/sanity/selectors.rb' - - 'qa/qa/service/docker_run/gitlab_runner.rb' - - 'qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb' - 'qa/qa/specs/features/browser_ui/3_create/web_ide/web_terminal_spec.rb' - 'qa/qa/specs/features/browser_ui/4_verify/testing/view_code_coverage_spec.rb' - 'qa/qa/specs/features/browser_ui/5_package/package_registry/generic_repository_spec.rb' - 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_group_level_spec.rb' - 'qa/qa/specs/features/browser_ui/5_package/package_registry/nuget/nuget_project_level_spec.rb' - 'rubocop/cop/database/multiple_databases.rb' + - 'rubocop/cop/database/rescue_query_canceled.rb' + - 'rubocop/cop/database/rescue_statement_timeout.rb' - 'rubocop/cop/default_scope.rb' - 'rubocop/cop/file_decompression.rb' - 'rubocop/cop/gitlab/httparty.rb' @@ -61,6 +59,7 @@ Naming/HeredocDelimiterNaming: - 'rubocop/cop/gitlab/module_with_instance_variables.rb' - 'rubocop/cop/gitlab/predicate_memoization.rb' - 'spec/controllers/projects/pipelines_controller_spec.rb' + - 'spec/db/docs_spec.rb' - 'spec/deprecation_toolkit_env.rb' - 'spec/factories/packages/debian/distribution.rb' - 'spec/factories/packages/debian/file_metadatum.rb' @@ -68,6 +67,7 @@ Naming/HeredocDelimiterNaming: - 'spec/features/task_lists_spec.rb' - 'spec/helpers/markup_helper_spec.rb' - 'spec/initializers/100_patch_omniauth_oauth2_spec.rb' + - 'spec/initializers/net_http_response_patch_spec.rb' - 'spec/initializers/rack_multipart_patch_spec.rb' - 'spec/initializers/secret_token_spec.rb' - 'spec/initializers/validate_database_config_spec.rb' @@ -82,6 +82,7 @@ Naming/HeredocDelimiterNaming: - 'spec/lib/gitlab/ci/yaml_processor_spec.rb' - 'spec/lib/gitlab/cluster/mixins/puma_cluster_spec.rb' - 'spec/lib/gitlab/conflict/file_collection_spec.rb' + - 'spec/lib/gitlab/database/loose_foreign_keys_spec.rb' - 'spec/lib/gitlab/diff/file_spec.rb' - 'spec/lib/gitlab/diff/pair_selector_spec.rb' - 'spec/lib/gitlab/diff/parser_spec.rb' @@ -131,10 +132,10 @@ Naming/HeredocDelimiterNaming: - 'spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb' - 'spec/services/google_cloud/generate_pipeline_service_spec.rb' - 'spec/services/task_list_toggle_service_spec.rb' - - 'spec/support/helpers/seed_helper.rb' - 'spec/support/helpers/stub_object_storage.rb' - 'spec/support/shared_examples/helm_commands_shared_examples.rb' - 'spec/support/shared_examples/models/taskable_shared_examples.rb' - 'spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb' - 'spec/support/test_reports/test_reports_helper.rb' + - 'spec/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences_rake_spec.rb' - 'spec/workers/post_receive_spec.rb' diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index d0d19211dea..6db5c13e8b5 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -3d769a33712796de113fe07647597a63d762c305 +993d944ebeea3e0ec8157481f125f9c70161ae4a diff --git a/Gemfile b/Gemfile index eed4dce2361..e619ed26e5b 100644 --- a/Gemfile +++ b/Gemfile @@ -78,7 +78,7 @@ gem 'u2f', '~> 0.2.1' gem 'validates_hostname', '~> 1.0.11' gem 'rubyzip', '~> 2.3.2', require: 'zip' # GitLab Pages letsencrypt support -gem 'acme-client', '~> 2.0', '>= 2.0.9' +gem 'acme-client', '~> 2.0' # Browser detection gem 'browser', '~> 4.2' @@ -435,6 +435,8 @@ group :test do gem 'capybara-screenshot', '~> 1.0.22' gem 'selenium-webdriver', '~> 3.142' + gem 'graphlyte', '~> 1.0.0' + gem 'shoulda-matchers', '~> 5.1.0', require: false gem 'email_spec', '~> 2.2.0' gem 'webmock', '~> 3.9.1' diff --git a/Gemfile.lock b/Gemfile.lock index e029123b3eb..ba7036da8ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -43,8 +43,9 @@ GEM remote: https://rubygems.org/ specs: RedCloth (4.3.2) - acme-client (2.0.9) - faraday (>= 0.17, < 2.0.0) + acme-client (2.0.11) + faraday (>= 1.0, < 3.0.0) + faraday-retry (~> 1.0) actioncable (6.1.6.1) actionpack (= 6.1.6.1) activesupport (= 6.1.6.1) @@ -615,6 +616,7 @@ GEM faraday (>= 1.0) faraday_middleware graphql-client + graphlyte (1.0.0) graphql (1.13.12) graphql-client (0.17.0) activesupport (>= 3.0) @@ -1481,7 +1483,7 @@ PLATFORMS DEPENDENCIES RedCloth (~> 4.3.2) - acme-client (~> 2.0, >= 2.0.9) + acme-client (~> 2.0) activerecord-explain-analyze (~> 0.1) acts-as-taggable-on (~> 9.0) addressable (~> 2.8) @@ -1593,6 +1595,7 @@ DEPENDENCIES grape_logging (~> 1.8) graphiql-rails (~> 1.8) graphlient (~> 0.5.0) + graphlyte (~> 1.0.0) graphql (~> 1.13.12) graphql-docs (~> 2.1.0) grpc (~> 1.42.0) diff --git a/app/assets/javascripts/runner/graphql/edit/runner_fields_shared.fragment.graphql b/app/assets/javascripts/runner/graphql/edit/runner_fields_shared.fragment.graphql index f900a0450e5..29abddf84f5 100644 --- a/app/assets/javascripts/runner/graphql/edit/runner_fields_shared.fragment.graphql +++ b/app/assets/javascripts/runner/graphql/edit/runner_fields_shared.fragment.graphql @@ -1,5 +1,4 @@ fragment RunnerFieldsShared on CiRunner { - __typename id shortSha runnerType diff --git a/app/assets/javascripts/runner/graphql/list/all_runners.query.graphql b/app/assets/javascripts/runner/graphql/list/all_runners.query.graphql index 6bb896dda16..7b2617f6c3c 100644 --- a/app/assets/javascripts/runner/graphql/list/all_runners.query.graphql +++ b/app/assets/javascripts/runner/graphql/list/all_runners.query.graphql @@ -31,7 +31,6 @@ query getAllRunners( editAdminUrl } pageInfo { - __typename ...PageInfo } } diff --git a/app/assets/javascripts/runner/graphql/list/group_runners.query.graphql b/app/assets/javascripts/runner/graphql/list/group_runners.query.graphql index 8755636a7ad..b7a8889ea0e 100644 --- a/app/assets/javascripts/runner/graphql/list/group_runners.query.graphql +++ b/app/assets/javascripts/runner/graphql/list/group_runners.query.graphql @@ -36,7 +36,6 @@ query getGroupRunners( } } pageInfo { - __typename ...PageInfo } } diff --git a/app/assets/javascripts/runner/graphql/list/list_item_shared.fragment.graphql b/app/assets/javascripts/runner/graphql/list/list_item_shared.fragment.graphql index 5e873135480..ce23bddb898 100644 --- a/app/assets/javascripts/runner/graphql/list/list_item_shared.fragment.graphql +++ b/app/assets/javascripts/runner/graphql/list/list_item_shared.fragment.graphql @@ -1,5 +1,4 @@ fragment ListItemShared on CiRunner { - __typename id description runnerType diff --git a/app/assets/javascripts/runner/graphql/show/runner_details_shared.fragment.graphql b/app/assets/javascripts/runner/graphql/show/runner_details_shared.fragment.graphql index b79ad4d9280..499c0156770 100644 --- a/app/assets/javascripts/runner/graphql/show/runner_details_shared.fragment.graphql +++ b/app/assets/javascripts/runner/graphql/show/runner_details_shared.fragment.graphql @@ -1,5 +1,4 @@ fragment RunnerDetailsShared on CiRunner { - __typename id shortSha runnerType diff --git a/app/views/projects/commits/_commits.html.haml b/app/views/projects/commits/_commits.html.haml index 764ddace0ad..bb3a38d6ac8 100644 --- a/app/views/projects/commits/_commits.html.haml +++ b/app/views/projects/commits/_commits.html.haml @@ -41,7 +41,7 @@ = n_('%s additional commit has been omitted to prevent performance issues.', '%s additional commits have been omitted to prevent performance issues.', hidden) % number_with_delimiter(hidden) - if can_update_merge_request && context_commits&.empty? - = render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5', data: { context_commits_empty: 'true' } }) do + = render Pajamas::ButtonComponent.new(button_options: { class: 'gl-mt-5 add-review-item-modal-trigger', data: { context_commits_empty: 'true' } }) do = _('Add previously merged commits') - if commits.size == 0 && context_commits.nil? diff --git a/config/feature_flags/development/changelog_commits_limitation.yml b/config/feature_flags/development/changelog_commits_limitation.yml index 3339fc7f946..ee1f0b0654d 100644 --- a/config/feature_flags/development/changelog_commits_limitation.yml +++ b/config/feature_flags/development/changelog_commits_limitation.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/364101 milestone: '15.1' type: development group: group::source code -default_enabled: false +default_enabled: true diff --git a/db/migrate/20220729073603_index_personal_access_tokens_on_id_and_created_at.rb b/db/migrate/20220729073603_index_personal_access_tokens_on_id_and_created_at.rb new file mode 100644 index 00000000000..b865f76fec1 --- /dev/null +++ b/db/migrate/20220729073603_index_personal_access_tokens_on_id_and_created_at.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class IndexPersonalAccessTokensOnIdAndCreatedAt < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_personal_access_tokens_on_id_and_created_at' + + disable_ddl_transaction! + + def up + add_concurrent_index :personal_access_tokens, [:id, :created_at], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :personal_access_tokens, INDEX_NAME + end +end diff --git a/db/schema_migrations/20220729073603 b/db/schema_migrations/20220729073603 new file mode 100644 index 00000000000..eb69bd1ce51 --- /dev/null +++ b/db/schema_migrations/20220729073603 @@ -0,0 +1 @@ +b0499c9b4cf3f39eec49dc7def7eaf8f1bbd03f2a34ba9eefa8440a109672136 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 75c00d0c09b..52fe23f0c45 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -29251,6 +29251,8 @@ CREATE INDEX index_path_locks_on_user_id ON path_locks USING btree (user_id); CREATE INDEX index_pe_approval_rules_on_required_approvals_and_created_at ON protected_environment_approval_rules USING btree (required_approvals, created_at); +CREATE INDEX index_personal_access_tokens_on_id_and_created_at ON personal_access_tokens USING btree (id, created_at); + CREATE UNIQUE INDEX index_personal_access_tokens_on_token_digest ON personal_access_tokens USING btree (token_digest); CREATE INDEX index_personal_access_tokens_on_user_id ON personal_access_tokens USING btree (user_id); diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 43981b602e3..31434a8ee6f 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -1048,7 +1048,8 @@ The [secure files API](../api/secure_files.md) enforces the following limits: ## Changelog API limits -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default. +> - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/33893) in GitLab 15.3. The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits: diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 7a3307e9548..1332eea26c0 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -384,7 +384,7 @@ POST /projects/:id/releases | `assets:links` | array of hash | no | An array of assets links. | | `assets:links:name`| string | required by: `assets:links` | The name of the link. Link names must be unique within the release. | | `assets:links:url` | string | required by: `assets:links` | The URL of the link. Link URLs must be unique within the release. | -| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). +| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | `released_at` | datetime | no | The date when the release is/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index c239d06ecfd..f9e07991948 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -93,14 +93,14 @@ Create an asset as a link from a Release. POST /projects/:id/releases/:tag_name/assets/links ``` -| Attribute | Type | Required | Description | -| ------------- | -------------- | -------- | ---------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). | -| `tag_name` | string | yes | The tag associated with the Release. | -| `name` | string | yes | The name of the link. Link names must be unique within the release. | -| `url` | string | yes | The URL of the link. Link URLs must be unique within the release. | -| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). | -| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | +| Attribute | Type | Required | Description | +|-------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). | +| `tag_name` | string | yes | The tag associated with the Release. | +| `name` | string | yes | The name of the link. Link names must be unique in the release. | +| `url` | string | yes | The URL of the link. Link URLs must be unique in the release. | +| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | +| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | Example request: @@ -141,7 +141,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id | `link_id` | integer | yes | The ID of the link. | | `name` | string | no | The name of the link. | | `url` | string | no | The URL of the link. | -| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). +| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | NOTE: diff --git a/doc/architecture/blueprints/database_scaling/size-limits.md b/doc/architecture/blueprints/database_scaling/size-limits.md index 45c3f4a659b..284f6402d3c 100644 --- a/doc/architecture/blueprints/database_scaling/size-limits.md +++ b/doc/architecture/blueprints/database_scaling/size-limits.md @@ -136,7 +136,7 @@ As such, the target size of a physical table after refactoring depends on the si There is no standard solution to reduce table sizes - there are many! 1. **Retention**: Delete unnecessary data, for example expire old and unneeded records. -1. **Remove STI**: We still use [single-table inheritance](../../../development/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables. +1. **Remove STI**: We still use [single-table inheritance](../../../development/database/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables. 1. **Index optimization**: Drop unnecessary indexes and consolidate overlapping indexes if possible. 1. **Optimise data types**: Review data type decisions and optimise data types where possible (example: use integer instead of text for an enum column) 1. **Partitioning**: Apply a partitioning scheme if there is a common access dimension. diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index cdf034d077a..9a65235fd05 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -2991,7 +2991,7 @@ released_at: '2021-03-15T08:00:00Z' > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271454) in GitLab 13.12. -Use `release:assets:links` to include [asset links](../../user/project/releases/index.md#release-assets) in the release. +Use `release:assets:links` to include [asset links](../../user/project/releases/release_fields.md#release-assets) in the release. Requires `release-cli` version v0.4.0 or later. diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index e18f1205718..7ab846cce3e 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -1,410 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/adding_database_indexes.md' +remove_date: '2022-11-05' --- -# Adding Database Indexes +This document was moved to [another location](database/adding_database_indexes.md). -Indexes can be used to speed up database queries, but when should you add a new -index? Traditionally the answer to this question has been to add an index for -every column used for filtering or joining data. For example, consider the -following query: - -```sql -SELECT * -FROM projects -WHERE user_id = 2; -``` - -Here we are filtering by the `user_id` column and as such a developer may decide -to index this column. - -While in certain cases indexing columns using the above approach may make sense, -it can actually have a negative impact. Whenever you write data to a table, any -existing indexes must also be updated. The more indexes there are, the slower this -can potentially become. Indexes can also take up significant disk space, depending -on the amount of data indexed and the index type. For example, PostgreSQL offers -`GIN` indexes which can be used to index certain data types that cannot be -indexed by regular B-tree indexes. These indexes, however, generally take up more -data and are slower to update compared to B-tree indexes. - -Because of all this, it's important make the following considerations -when adding a new index: - -1. Do the new queries re-use as many existing indexes as possible? -1. Is there enough data that using an index is faster than iterating over - rows in the table? -1. Is the overhead of maintaining the index worth the reduction in query - timings? - -## Re-using Queries - -The first step is to make sure your query re-uses as many existing indexes as -possible. For example, consider the following query: - -```sql -SELECT * -FROM todos -WHERE user_id = 123 -AND state = 'open'; -``` - -Now imagine we already have an index on the `user_id` column but not on the -`state` column. One may think this query performs badly due to `state` being -unindexed. In reality the query may perform just fine given the index on -`user_id` can filter out enough rows. - -The best way to determine if indexes are re-used is to run your query using -`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering, -you may find an extra index doesn't make much, if any, difference. - -In short: - -1. Try to write your query in such a way that it re-uses as many existing - indexes as possible. -1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most - ideal query. - -## Data Size - -A database may not use an index even when a regular sequence scan -(iterating over all rows) is faster, especially for small tables. - -Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows. -You may _not_ want to add an index if the table size is small (<`1,000` records), -or if existing indexes already filter out enough rows. - -## Maintenance Overhead - -Indexes have to be updated on every table write. In the case of PostgreSQL, _all_ -existing indexes are updated whenever data is written to a table. As a -result, having many indexes on the same table slows down writes. It's therefore important -to balance query performance with the overhead of maintaining an extra index. - -Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases -INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth -it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE -timings are unaffected. - -## Finding Unused Indexes - -To see which indexes are unused you can run the following query: - -```sql -SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass)) -FROM pg_stat_all_indexes -WHERE schemaname = 'public' -AND idx_scan = 0 -AND idx_tup_read = 0 -AND idx_tup_fetch = 0 -ORDER BY pg_relation_size(indexrelname::regclass) desc; -``` - -This query outputs a list containing all indexes that are never used and sorts -them by indexes sizes in descending order. This query helps in -determining whether existing indexes are still required. More information on -the meaning of the various columns can be found at -. - -To determine if an index is still being used on production, use the following -Thanos query with your index name: - -```sql -sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) -``` - -Because the query output relies on the actual usage of your database, it -may be affected by factors such as: - -- Certain queries never being executed, thus not being able to use certain - indexes. -- Certain tables having little data, resulting in PostgreSQL using sequence - scans instead of index scans. - -This data is only reliable for a frequently used database with -plenty of data, and using as many GitLab features as possible. - -## Requirements for naming indexes - -Indexes with complex definitions must be explicitly named rather than -relying on the implicit naming behavior of migration methods. In short, -that means you **must** provide an explicit name argument for an index -created with one or more of the following options: - -- `where` -- `using` -- `order` -- `length` -- `type` -- `opclass` - -### Considerations for index names - -Check our [Constraints naming conventions](database/constraint_naming_convention.md) page. - -### Why explicit names are required - -As Rails is database agnostic, it generates an index name only -from the required options of all indexes: table name and column names. -For example, imagine the following two indexes are created in a migration: - -```ruby -def up - add_index :my_table, :my_column - - add_index :my_table, :my_column, where: 'my_column IS NOT NULL' -end -``` - -Creation of the second index would fail, because Rails would generate -the same name for both indexes. - -This naming issue is further complicated by the behavior of the `index_exists?` method. -It considers only the table name, column names, and uniqueness specification -of the index when making a comparison. Consider: - -```ruby -def up - unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL') - add_index :my_table, :my_column, where: 'my_column IS NOT NULL' - end -end -``` - -The call to `index_exists?` returns true if **any** index exists on -`:my_table` and `:my_column`, and index creation is bypassed. - -The `add_concurrent_index` helper is a requirement for creating indexes -on populated tables. Because it cannot be used inside a transactional -migration, it has a built-in check that detects if the index already -exists. In the event a match is found, index creation is skipped. -Without an explicit name argument, Rails can return a false positive -for `index_exists?`, causing a required index to not be created -properly. By always requiring a name for certain types of indexes, the -chance of error is greatly reduced. - -## Temporary indexes - -There may be times when an index is only needed temporarily. - -For example, in a migration, a column of a table might be conditionally -updated. To query which columns must be updated in the -[query performance guidelines](query_performance.md), an index is needed -that would otherwise not be used. - -In these cases, consider a temporary index. To specify a -temporary index: - -1. Prefix the index name with `tmp_` and follow the [naming conventions](database/constraint_naming_convention.md). -1. Create a follow-up issue to remove the index in the next (or future) milestone. -1. Add a comment in the migration mentioning the removal issue. - -A temporary migration would look like: - -```ruby -INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled' - -def up - # Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234 - add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME -end - -def down - remove_concurrent_index_by_name :projects, INDEX_NAME -end -``` - -## Create indexes asynchronously - -For very large tables, index creation can be a challenge to manage. -While `add_concurrent_index` creates indexes in a way that does not block -normal traffic, it can still be problematic when index creation runs for -many hours. Necessary database operations like `autovacuum` cannot run, and -on GitLab.com, the deployment process is blocked waiting for index -creation to finish. - -To limit impact on GitLab.com, a process exists to create indexes -asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, -index creation can proceed at a lower level of risk. - -### Schedule index creation for a low-impact time - -1. [Schedule the index to be created](#schedule-the-index-to-be-created). -1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). -1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). - -### Schedule the index to be created - -Create an MR with a post-deployment migration which prepares the index -for asynchronous creation. An example of creating an index using -the asynchronous index helpers can be seen in the block below. This migration -enters the index name and definition into the `postgres_async_indexes` -table. The process that runs on weekends pulls indexes from this -table and attempt to create them. - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -def up - prepare_async_index :ci_builds, :some_column, name: INDEX_NAME -end - -def down - unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME -end -``` - -### Verify the MR was deployed and the index exists in production - -You can verify if the post-deploy migration was executed on GitLab.com by: - -- Executing `/chatops run auto_deploy status `. If the output returns `db/gprd`, - the post-deploy migration has been executed in the production database. More details in this - [guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). -- Use a meta-command in #database-lab, such as: `\d `. - - Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). -- Ask someone in #database to check if the index exists. -- With proper access, you can also verify directly on production or in a - production clone. - -### Add a migration to create the index synchronously - -After the index is verified to exist on the production database, create a second -merge request that adds the index synchronously. The schema changes must be -updated and committed to `structure.sql` in this second merge request. -The synchronous migration results in a no-op on GitLab.com, but you should still add the -migration as expected for other installations. The below block -demonstrates how to create the second migration for the previous -asynchronous example. - -**WARNING:** -Verify that the index exists in production before merging a second migration with `add_concurrent_index`. -If the second migration is deployed before the index has been created, -the index is created synchronously when the second migration executes. - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -disable_ddl_transaction! - -def up - add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME -end - -def down - remove_concurrent_index_by_name :ci_builds, INDEX_NAME -end -``` - -## Test database index changes locally - -You must test the database index changes locally before creating a merge request. - -### Verify indexes created asynchronously - -Use the asynchronous index helpers on your local environment to test changes for creating an index: - -1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console. -1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table. -1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously. -1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d ` to check that your newly created index exists. - -## Drop indexes asynchronously - -For very large tables, index destruction can be a challenge to manage. -While `remove_concurrent_index` removes indexes in a way that does not block -normal traffic, it can still be problematic if index destruction runs for -during autovacuum. Necessary database operations like `autovacuum` cannot run, and -the deployment process on GitLab.com is blocked while waiting for index -destruction to finish. - -To limit the impact on GitLab.com, use the following process to remove indexes -asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, -index destruction can proceed at a lower level of risk. - -1. [Schedule the index to be removed](#schedule-the-index-to-be-removed). -1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). -1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). - -### Schedule the index to be removed - -Create an MR with a post-deployment migration which prepares the index -for asynchronous destruction. For example. to destroy an index using -the asynchronous index helpers: - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -def up - prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME -end - -def down - unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME -end -``` - -This migration enters the index name and definition into the `postgres_async_indexes` -table. The process that runs on weekends pulls indexes from this table and attempt -to remove them. - -You must test the database index changes locally before creating a merge request. - -### Verify the MR was deployed and the index exists in production - -You can verify if the MR was deployed to GitLab.com with -`/chatops run auto_deploy status `. To verify the existence of -the index, you can: - -- Use a meta-command in `#database-lab`, for example: `\d `. - - Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). -- Ask someone in `#database` to check if the index exists. -- If you have access, you can verify directly on production or in a - production clone. - -### Add a migration to destroy the index synchronously - -After you verify the index exists in the production database, create a second -merge request that removes the index synchronously. The schema changes must be -updated and committed to `structure.sql` in this second merge request. -The synchronous migration results in a no-op on GitLab.com, but you should still add the -migration as expected for other installations. For example, to -create the second migration for the previous asynchronous example: - -**WARNING:** -Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`. -If the second migration is deployed before the index has been destroyed, -the index is destroyed synchronously when the second migration executes. - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -disable_ddl_transaction! - -def up - remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME -end - -def down - add_concurrent_index :ci_builds, :some_column, INDEX_NAME -end -``` - -### Verify indexes removed asynchronously - -To test changes for removing an index, use the asynchronous index helpers on your local environment: - -1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console. -1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table. -1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously. -1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) - command `gdk psql` and run `\d ` to check that the destroyed index no longer exists. + + + + diff --git a/doc/development/database/adding_database_indexes.md b/doc/development/database/adding_database_indexes.md new file mode 100644 index 00000000000..8e390f47332 --- /dev/null +++ b/doc/development/database/adding_database_indexes.md @@ -0,0 +1,410 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Adding Database Indexes + +Indexes can be used to speed up database queries, but when should you add a new +index? Traditionally the answer to this question has been to add an index for +every column used for filtering or joining data. For example, consider the +following query: + +```sql +SELECT * +FROM projects +WHERE user_id = 2; +``` + +Here we are filtering by the `user_id` column and as such a developer may decide +to index this column. + +While in certain cases indexing columns using the above approach may make sense, +it can actually have a negative impact. Whenever you write data to a table, any +existing indexes must also be updated. The more indexes there are, the slower this +can potentially become. Indexes can also take up significant disk space, depending +on the amount of data indexed and the index type. For example, PostgreSQL offers +`GIN` indexes which can be used to index certain data types that cannot be +indexed by regular B-tree indexes. These indexes, however, generally take up more +data and are slower to update compared to B-tree indexes. + +Because of all this, it's important make the following considerations +when adding a new index: + +1. Do the new queries re-use as many existing indexes as possible? +1. Is there enough data that using an index is faster than iterating over + rows in the table? +1. Is the overhead of maintaining the index worth the reduction in query + timings? + +## Re-using Queries + +The first step is to make sure your query re-uses as many existing indexes as +possible. For example, consider the following query: + +```sql +SELECT * +FROM todos +WHERE user_id = 123 +AND state = 'open'; +``` + +Now imagine we already have an index on the `user_id` column but not on the +`state` column. One may think this query performs badly due to `state` being +unindexed. In reality the query may perform just fine given the index on +`user_id` can filter out enough rows. + +The best way to determine if indexes are re-used is to run your query using +`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering, +you may find an extra index doesn't make much, if any, difference. + +In short: + +1. Try to write your query in such a way that it re-uses as many existing + indexes as possible. +1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most + ideal query. + +## Data Size + +A database may not use an index even when a regular sequence scan +(iterating over all rows) is faster, especially for small tables. + +Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows. +You may _not_ want to add an index if the table size is small (<`1,000` records), +or if existing indexes already filter out enough rows. + +## Maintenance Overhead + +Indexes have to be updated on every table write. In the case of PostgreSQL, _all_ +existing indexes are updated whenever data is written to a table. As a +result, having many indexes on the same table slows down writes. It's therefore important +to balance query performance with the overhead of maintaining an extra index. + +Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases +INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth +it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE +timings are unaffected. + +## Finding Unused Indexes + +To see which indexes are unused you can run the following query: + +```sql +SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass)) +FROM pg_stat_all_indexes +WHERE schemaname = 'public' +AND idx_scan = 0 +AND idx_tup_read = 0 +AND idx_tup_fetch = 0 +ORDER BY pg_relation_size(indexrelname::regclass) desc; +``` + +This query outputs a list containing all indexes that are never used and sorts +them by indexes sizes in descending order. This query helps in +determining whether existing indexes are still required. More information on +the meaning of the various columns can be found at +. + +To determine if an index is still being used on production, use the following +Thanos query with your index name: + +```sql +sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) +``` + +Because the query output relies on the actual usage of your database, it +may be affected by factors such as: + +- Certain queries never being executed, thus not being able to use certain + indexes. +- Certain tables having little data, resulting in PostgreSQL using sequence + scans instead of index scans. + +This data is only reliable for a frequently used database with +plenty of data, and using as many GitLab features as possible. + +## Requirements for naming indexes + +Indexes with complex definitions must be explicitly named rather than +relying on the implicit naming behavior of migration methods. In short, +that means you **must** provide an explicit name argument for an index +created with one or more of the following options: + +- `where` +- `using` +- `order` +- `length` +- `type` +- `opclass` + +### Considerations for index names + +Check our [Constraints naming conventions](constraint_naming_convention.md) page. + +### Why explicit names are required + +As Rails is database agnostic, it generates an index name only +from the required options of all indexes: table name and column names. +For example, imagine the following two indexes are created in a migration: + +```ruby +def up + add_index :my_table, :my_column + + add_index :my_table, :my_column, where: 'my_column IS NOT NULL' +end +``` + +Creation of the second index would fail, because Rails would generate +the same name for both indexes. + +This naming issue is further complicated by the behavior of the `index_exists?` method. +It considers only the table name, column names, and uniqueness specification +of the index when making a comparison. Consider: + +```ruby +def up + unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL') + add_index :my_table, :my_column, where: 'my_column IS NOT NULL' + end +end +``` + +The call to `index_exists?` returns true if **any** index exists on +`:my_table` and `:my_column`, and index creation is bypassed. + +The `add_concurrent_index` helper is a requirement for creating indexes +on populated tables. Because it cannot be used inside a transactional +migration, it has a built-in check that detects if the index already +exists. In the event a match is found, index creation is skipped. +Without an explicit name argument, Rails can return a false positive +for `index_exists?`, causing a required index to not be created +properly. By always requiring a name for certain types of indexes, the +chance of error is greatly reduced. + +## Temporary indexes + +There may be times when an index is only needed temporarily. + +For example, in a migration, a column of a table might be conditionally +updated. To query which columns must be updated in the +[query performance guidelines](../query_performance.md), an index is needed +that would otherwise not be used. + +In these cases, consider a temporary index. To specify a +temporary index: + +1. Prefix the index name with `tmp_` and follow the [naming conventions](constraint_naming_convention.md). +1. Create a follow-up issue to remove the index in the next (or future) milestone. +1. Add a comment in the migration mentioning the removal issue. + +A temporary migration would look like: + +```ruby +INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled' + +def up + # Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234 + add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME +end + +def down + remove_concurrent_index_by_name :projects, INDEX_NAME +end +``` + +## Create indexes asynchronously + +For very large tables, index creation can be a challenge to manage. +While `add_concurrent_index` creates indexes in a way that does not block +normal traffic, it can still be problematic when index creation runs for +many hours. Necessary database operations like `autovacuum` cannot run, and +on GitLab.com, the deployment process is blocked waiting for index +creation to finish. + +To limit impact on GitLab.com, a process exists to create indexes +asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, +index creation can proceed at a lower level of risk. + +### Schedule index creation for a low-impact time + +1. [Schedule the index to be created](#schedule-the-index-to-be-created). +1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). +1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). + +### Schedule the index to be created + +Create an MR with a post-deployment migration which prepares the index +for asynchronous creation. An example of creating an index using +the asynchronous index helpers can be seen in the block below. This migration +enters the index name and definition into the `postgres_async_indexes` +table. The process that runs on weekends pulls indexes from this +table and attempt to create them. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +def up + prepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end + +def down + unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end +``` + +### Verify the MR was deployed and the index exists in production + +You can verify if the post-deploy migration was executed on GitLab.com by: + +- Executing `/chatops run auto_deploy status `. If the output returns `db/gprd`, + the post-deploy migration has been executed in the production database. More details in this + [guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). +- Use a meta-command in #database-lab, such as: `\d `. + - Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). +- Ask someone in #database to check if the index exists. +- With proper access, you can also verify directly on production or in a + production clone. + +### Add a migration to create the index synchronously + +After the index is verified to exist on the production database, create a second +merge request that adds the index synchronously. The schema changes must be +updated and committed to `structure.sql` in this second merge request. +The synchronous migration results in a no-op on GitLab.com, but you should still add the +migration as expected for other installations. The below block +demonstrates how to create the second migration for the previous +asynchronous example. + +**WARNING:** +Verify that the index exists in production before merging a second migration with `add_concurrent_index`. +If the second migration is deployed before the index has been created, +the index is created synchronously when the second migration executes. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +disable_ddl_transaction! + +def up + add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME +end + +def down + remove_concurrent_index_by_name :ci_builds, INDEX_NAME +end +``` + +## Test database index changes locally + +You must test the database index changes locally before creating a merge request. + +### Verify indexes created asynchronously + +Use the asynchronous index helpers on your local environment to test changes for creating an index: + +1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console. +1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table. +1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously. +1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d ` to check that your newly created index exists. + +## Drop indexes asynchronously + +For very large tables, index destruction can be a challenge to manage. +While `remove_concurrent_index` removes indexes in a way that does not block +normal traffic, it can still be problematic if index destruction runs for +during autovacuum. Necessary database operations like `autovacuum` cannot run, and +the deployment process on GitLab.com is blocked while waiting for index +destruction to finish. + +To limit the impact on GitLab.com, use the following process to remove indexes +asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, +index destruction can proceed at a lower level of risk. + +1. [Schedule the index to be removed](#schedule-the-index-to-be-removed). +1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). +1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). + +### Schedule the index to be removed + +Create an MR with a post-deployment migration which prepares the index +for asynchronous destruction. For example. to destroy an index using +the asynchronous index helpers: + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +def up + prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME +end + +def down + unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end +``` + +This migration enters the index name and definition into the `postgres_async_indexes` +table. The process that runs on weekends pulls indexes from this table and attempt +to remove them. + +You must test the database index changes locally before creating a merge request. + +### Verify the MR was deployed and the index exists in production + +You can verify if the MR was deployed to GitLab.com with +`/chatops run auto_deploy status `. To verify the existence of +the index, you can: + +- Use a meta-command in `#database-lab`, for example: `\d `. + - Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). +- Ask someone in `#database` to check if the index exists. +- If you have access, you can verify directly on production or in a + production clone. + +### Add a migration to destroy the index synchronously + +After you verify the index exists in the production database, create a second +merge request that removes the index synchronously. The schema changes must be +updated and committed to `structure.sql` in this second merge request. +The synchronous migration results in a no-op on GitLab.com, but you should still add the +migration as expected for other installations. For example, to +create the second migration for the previous asynchronous example: + +**WARNING:** +Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`. +If the second migration is deployed before the index has been destroyed, +the index is destroyed synchronously when the second migration executes. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +disable_ddl_transaction! + +def up + remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME +end + +def down + add_concurrent_index :ci_builds, :some_column, INDEX_NAME +end +``` + +### Verify indexes removed asynchronously + +To test changes for removing an index, use the asynchronous index helpers on your local environment: + +1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console. +1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table. +1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously. +1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) + command `gdk psql` and run `\d ` to check that the destroyed index no longer exists. diff --git a/doc/development/database/database_query_comments.md b/doc/development/database/database_query_comments.md new file mode 100644 index 00000000000..2798071bc06 --- /dev/null +++ b/doc/development/database/database_query_comments.md @@ -0,0 +1,62 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Database query comments with Marginalia + +The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add +query comments containing application related context information to PostgreSQL +queries generated by ActiveRecord. + +It is very useful for tracing problematic queries back to the application source. + +An engineer during an on-call incident has the full context of a query +and its application source from the comments. + +## Metadata information in comments + +Queries generated from **Rails** include the following metadata in comments: + +- `application` +- `correlation_id` +- `endpoint_id` +- `line` + +Queries generated from **Sidekiq** workers include the following metadata +in comments: + +- `application` +- `jid` +- `correlation_id` +- `endpoint_id` +- `line` + +`endpoint_id` is a single field that can represent any endpoint in the application: + +- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`. +- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`. +- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`. + +`line` is not present in production logs due to the additional overhead required. + +Examples of queries with comments: + +- Rails: + + ```sql + /*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1 + ``` + +- Grape: + + ```sql + /*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1 + ``` + +- Sidekiq: + + ```sql + /*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3 + ``` diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 1b8dead942c..553c9811ee9 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -36,17 +36,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Debugging -- Tracing the source of an SQL query using query comments with [Marginalia](../database_query_comments.md) +- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md) - Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) ## Best practices -- [Adding database indexes](../adding_database_indexes.md) +- [Adding database indexes](adding_database_indexes.md) - [Foreign keys & associations](../foreign_keys.md) - [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md) - [`NOT NULL` constraints](not_null_constraints.md) - [Strings and the Text data type](strings_and_the_text_data_type.md) -- [Single table inheritance](../single_table_inheritance.md) +- [Single table inheritance](single_table_inheritance.md) - [Polymorphic associations](polymorphic_associations.md) - [Serializing data](../serializing_data.md) - [Hash indexes](../hash_indexes.md) diff --git a/doc/development/single_table_inheritance.md b/doc/development/database/single_table_inheritance.md similarity index 100% rename from doc/development/single_table_inheritance.md rename to doc/development/database/single_table_inheritance.md diff --git a/doc/development/database_query_comments.md b/doc/development/database_query_comments.md index 2798071bc06..7f9def7e567 100644 --- a/doc/development/database_query_comments.md +++ b/doc/development/database_query_comments.md @@ -1,62 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/database_query_comments.md' +remove_date: '2022-11-05' --- -# Database query comments with Marginalia +This document was moved to [another location](database/database_query_comments.md). -The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add -query comments containing application related context information to PostgreSQL -queries generated by ActiveRecord. - -It is very useful for tracing problematic queries back to the application source. - -An engineer during an on-call incident has the full context of a query -and its application source from the comments. - -## Metadata information in comments - -Queries generated from **Rails** include the following metadata in comments: - -- `application` -- `correlation_id` -- `endpoint_id` -- `line` - -Queries generated from **Sidekiq** workers include the following metadata -in comments: - -- `application` -- `jid` -- `correlation_id` -- `endpoint_id` -- `line` - -`endpoint_id` is a single field that can represent any endpoint in the application: - -- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`. -- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`. -- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`. - -`line` is not present in production logs due to the additional overhead required. - -Examples of queries with comments: - -- Rails: - - ```sql - /*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1 - ``` - -- Grape: - - ```sql - /*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1 - ``` - -- Sidekiq: - - ```sql - /*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3 - ``` + + + + diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index d8b6456469a..de91bdb0780 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -120,7 +120,7 @@ columns manually for existing tables as this causes confusion to other people using `db/structure.sql` generated by Rails. NOTE: -[Creating an index asynchronously requires two merge requests.](adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously) +[Creating an index asynchronously requires two merge requests.](database/adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously) When done, commit the schema change in the merge request that adds the index with `add_concurrent_index`. @@ -612,7 +612,7 @@ might not be required, like: Additionally, wide indexes are not required to match all filter criteria of queries, we just need to cover enough columns so that the index lookup has a small enough selectivity. Please review our -[Adding Database indexes](adding_database_indexes.md) guide for more details. +[Adding Database indexes](database/adding_database_indexes.md) guide for more details. When adding an index to a non-empty table make sure to use the method `add_concurrent_index` instead of the regular `add_index` method. @@ -641,7 +641,7 @@ end You must explicitly name indexes that are created with more complex definitions beyond table name, column names, and uniqueness constraint. -Consult the [Adding Database Indexes](adding_database_indexes.md#requirements-for-naming-indexes) +Consult the [Adding Database Indexes](database/adding_database_indexes.md#requirements-for-naming-indexes) guide for more details. If you need to add a unique index, please keep in mind there is the possibility @@ -659,7 +659,7 @@ If a migration requires conditional logic based on the absence or presence of an index, you must test for existence of that index using its name. This helps avoids problems with how Rails compares index definitions, which can lead to unexpected results. For more details, review the -[Adding Database Indexes](adding_database_indexes.md#why-explicit-names-are-required) +[Adding Database Indexes](database/adding_database_indexes.md#why-explicit-names-are-required) guide. The easiest way to test for existence of an index by name is to use the @@ -1198,7 +1198,7 @@ If using a model in the migrations, you should first using `reset_column_information`. If using a model that leverages single table inheritance (STI), there are [special -considerations](single_table_inheritance.md#in-migrations). +considerations](database/single_table_inheritance.md#in-migrations). This avoids problems where a column that you are using was altered and cached in a previous migration. diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index e8ccb24e2c5..c7d18353c47 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -32,12 +32,10 @@ When you create a release, or after, you can: - Add release notes. - Add a message for the Git tag associated with the release. - [Associate milestones with it](#associate-milestones-with-a-release). -- Attach [release assets](#release-assets), like runbooks or packages. +- Attach [release assets](release_fields.md#release-assets), like runbooks or packages. ## View releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8. - To view a list of releases: - On the left sidebar, select **Deployments > Releases**, or @@ -81,18 +79,18 @@ To create a release in the Releases page: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Deployments > Releases** and select **New release**. -1. From the [**Tag name**](#tag-name) dropdown, either: +1. From the [**Tag name**](release_fields.md#tag-name) dropdown, either: - Select an existing Git tag. Selecting an existing tag that is already associated with a release results in a validation error. - Enter a new Git tag name. 1. From the **Create from** dropdown, select a branch or commit SHA to use when creating the new tag. 1. Optional. Enter additional information about the release, including: - - [Title](#title). + - [Title](release_fields.md#title). - [Milestones](#associate-milestones-with-a-release). - - [Release notes](#release-notes-description). + - [Release notes](release_fields.md#release-notes-description). - Whether or not to include the [Tag message](../../../topics/git/tags.md). - - [Asset links](#links). + - [Asset links](release_fields.md#links). 1. Select **Create release**. ### Create a release in the Tags page @@ -298,9 +296,6 @@ is not available. ## Edit a release -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. -> - Asset link editing [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. - Only users with at least the Developer role can edit releases. Read more about [Release permissions](#release-permissions). @@ -428,277 +423,6 @@ complete overlapping period. For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md). -## Release fields - -The following fields are available when you create or edit a release. - -### Title - -The release title can be customized using the **Release title** field when -creating or editing a release. If no title is provided, the release's tag name -is used instead. - -### Tag name - -The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) -for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the -[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). - -For example, for GitLab version `10.5.7`: - -- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. -- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. -- `7` represents the patch number. - -Any part of the version number can be multiple digits, for example, `13.10.11`. - -### Release notes description - -Every release has a description. You can add any text you like, but we recommend -including a changelog to describe the content of your release. This helps users -quickly scan the differences between each release you publish. - -[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can -be included in Release note descriptions by selecting **Include tag message in -the release notes**. - -Description supports [Markdown](../../markdown.md). - -### Release assets - -A release contains the following types of assets: - -- [Source code](#source-code) -- [Link](#links) - -#### Source code - -GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar` -archived source code from the given Git tag. These are read-only assets. - -#### Links - -A link is any URL which can point to whatever you like: documentation, built -binaries, or other related materials. These can be both internal or external -links from your GitLab instance. -Each link as an asset has the following attributes: - -| Attribute | Description | Required | -| ---- | ----------- | --- | -| `name` | The name of the link. | Yes | -| `url` | The URL to download a file. | Yes | -| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No | -| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No | - -##### Permanent link to latest release - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. - -Latest release page is accessible through a permanent URL. -GitLab will redirect to the latest release page URL when it is visited. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/permalink/latest -``` - -We also support, suffix path carry forward on the redirect to the latest release. -Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`. - -Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage. - -###### Sorting preferences - -By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945). - -##### Permanent links to release assets - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. - -The assets associated with a release are accessible through a permanent URL. -GitLab always redirects this URL to the actual asset -location, so even if the assets move to a different location, you can continue -to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/:release/downloads/:filepath -``` - -If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` -namespace and `gitlab-runner` project on `gitlab.com`, for example: - -```json -{ - "name": "linux amd64", - "filepath": "/binaries/gitlab-runner-linux-amd64", - "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", - "link_type": "other" -} -``` - -This asset has a direct link of: - -```plaintext -https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 -``` - -The physical location of the asset can change at any time and the direct link remains unchanged. - -##### Permanent links to latest release assets - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. - -The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath -``` - -If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org` -namespace and `gitlab-runner` project on `gitlab.com`, for example: - -```json -{ - "name": "linux amd64", - "filepath": "/binaries/gitlab-runner-linux-amd64", - "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", - "link_type": "other" -} -``` - -This asset has a direct link of: - -```plaintext -https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64 -``` - -##### Link Types - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1. - -The four types of links are "Runbook," "Package," "Image," and "Other." -The `link_type` parameter accepts one of the following four values: - -- `runbook` -- `package` -- `image` -- `other` (default) - -This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. - -##### Use a generic package for attaching binaries - -You can use [generic packages](../../packages/generic_packages/index.md) -to store any artifacts from a release or tag pipeline, -that can also be used for attaching binary files to an individual release entry. -You basically need to: - -1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file). -1. [Attach the package link to the release](#links). - -The following example generates release assets, publishes them -as a generic package, and then creates a release: - -```yaml -stages: - - build - - upload - - release - -variables: - # Package version can only contain numbers (0-9), and dots (.). - # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion. - # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file - PACKAGE_VERSION: "1.2.3" - DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}" - LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}" - PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}" - -build: - stage: build - image: alpine:latest - rules: - - if: $CI_COMMIT_TAG - script: - - mkdir bin - - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY} - - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY} - artifacts: - paths: - - bin/ - -upload: - stage: upload - image: curlimages/curl:latest - rules: - - if: $CI_COMMIT_TAG - script: - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}" - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}" - -release: - # Caution, as of 2021-02-02 these assets links require a login, see: - # https://gitlab.com/gitlab-org/gitlab/-/issues/299384 - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - rules: - - if: $CI_COMMIT_TAG - script: - - | - release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ - --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \ - --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}" -``` - -PowerShell users may need to escape the double quote `"` inside a JSON -string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json` -before passing on to the `release-cli`. -For example: - -```yaml -release: - script: - - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}" - - $env:assetjson = $env:asset | ConvertTo-Json - - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson -``` - -NOTE: -Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md) -links to a release is not recommended, because artifacts are ephemeral and -are used to pass data in the same pipeline. This means there's a risk that -they could either expire or someone might manually delete them. - -#### Number of new and total features **(FREE SAAS)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5. - -On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project. - -![Feature count](img/feature_count_v14_6.png "Number of features in a release") - -The totals are displayed on [shields](https://shields.io/) and are generated per release by -[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake). - -| Item | Formula | -| ------ | ------ | -| `New features` | Total count of release posts across all tiers for a single release in the project. | -| `Total features` | Total count of release posts in reverse order for all releases in the project. | - -The counts are also shown by license tier. - -| Item | Formula | -| ------ | ------ | -| `New features` | Total count of release posts across a single tier for a single release in the project. | -| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. | - ## Release evidence > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6. @@ -847,7 +571,7 @@ In the API: - Users with the Guest role have read and download access to the project releases. This includes associated Git-tag-names, release description, author information of the releases. - However, other repository-related information, such as [source code](#source-code), [release evidence](#release-evidence) are redacted. + However, other repository-related information, such as [source code](release_fields.md#source-code), [release evidence](#release-evidence) are redacted. ### Create, update, and delete a release and its assets diff --git a/doc/user/project/releases/release_fields.md b/doc/user/project/releases/release_fields.md new file mode 100644 index 00000000000..647cac9c38e --- /dev/null +++ b/doc/user/project/releases/release_fields.md @@ -0,0 +1,274 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Release fields + +The following fields are available when you create or edit a release. + +## Title + +The release title can be customized using the **Release title** field when +creating or editing a release. If no title is provided, the release's tag name +is used instead. + +## Tag name + +The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) +for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the +[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). + +For example, for GitLab version `10.5.7`: + +- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. +- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. +- `7` represents the patch number. + +Any part of the version number can be multiple digits, for example, `13.10.11`. + +## Release notes description + +Every release has a description. You can add any text you like, but we recommend +including a changelog to describe the content of your release. This helps users +quickly scan the differences between each release you publish. + +[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can +be included in Release note descriptions by selecting **Include tag message in +the release notes**. + +Description supports [Markdown](../../markdown.md). + +## Release assets + +A release contains the following types of assets: + +- [Source code](#source-code) +- [Link](#links) + +### Source code + +GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar` +archived source code from the given Git tag. These are read-only assets. + +### Links + +A link is any URL which can point to whatever you like: documentation, built +binaries, or other related materials. These can be both internal or external +links from your GitLab instance. +Each link as an asset has the following attributes: + +| Attribute | Description | Required | +|-------------|--------------------------------------------------------------------------------------------------------------|----------| +| `name` | The name of the link. | Yes | +| `url` | The URL to download a file. | Yes | +| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No | +| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No | + +#### Permanent link to latest release + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. + +Latest release page is accessible through a permanent URL. +GitLab redirects to the latest release page URL when it is visited. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/permalink/latest +``` + +We also support, suffix path carry forward on the redirect to the latest release. +Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`. + +Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage. + +##### Sorting preferences + +By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945). + +#### Permanent links to release assets + +The assets associated with a release are accessible through a permanent URL. +GitLab always redirects this URL to the actual asset +location, so even if the assets move to a different location, you can continue +to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/:release/downloads/:filepath +``` + +If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` +namespace and `gitlab-runner` project on `gitlab.com`, for example: + +```json +{ + "name": "linux amd64", + "filepath": "/binaries/gitlab-runner-linux-amd64", + "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", + "link_type": "other" +} +``` + +This asset has a direct link of: + +```plaintext +https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 +``` + +The physical location of the asset can change at any time and the direct link remains unchanged. + +#### Permanent links to latest release assets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. + +The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath +``` + +If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org` +namespace and `gitlab-runner` project on `gitlab.com`, for example: + +```json +{ + "name": "linux amd64", + "filepath": "/binaries/gitlab-runner-linux-amd64", + "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", + "link_type": "other" +} +``` + +This asset has a direct link of: + +```plaintext +https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64 +``` + +#### Link Types + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1. + +The four types of links are "Runbook," "Package," "Image," and "Other." +The `link_type` parameter accepts one of the following four values: + +- `runbook` +- `package` +- `image` +- `other` (default) + +This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. + +#### Use a generic package for attaching binaries + +You can use [generic packages](../../packages/generic_packages/index.md) +to store any artifacts from a release or tag pipeline, +that can also be used for attaching binary files to an individual release entry. +You basically need to: + +1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file). +1. [Attach the package link to the release](#links). + +The following example generates release assets, publishes them +as a generic package, and then creates a release: + +```yaml +stages: + - build + - upload + - release + +variables: + # Package version can only contain numbers (0-9), and dots (.). + # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion. + # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file + PACKAGE_VERSION: "1.2.3" + DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}" + LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}" + PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}" + +build: + stage: build + image: alpine:latest + rules: + - if: $CI_COMMIT_TAG + script: + - mkdir bin + - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY} + - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY} + artifacts: + paths: + - bin/ + +upload: + stage: upload + image: curlimages/curl:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}" + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}" + +release: + # Caution, as of 2021-02-02 these assets links require a login, see: + # https://gitlab.com/gitlab-org/gitlab/-/issues/299384 + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ + --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \ + --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}" +``` + +PowerShell users may need to escape the double quote `"` inside a JSON +string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json` +before passing on to the `release-cli`. +For example: + +```yaml +release: + script: + - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}" + - $env:assetjson = $env:asset | ConvertTo-Json + - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson +``` + +NOTE: +Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md) +links to a release is not recommended, because artifacts are ephemeral and +are used to pass data in the same pipeline. This means there's a risk that +they could either expire or someone might manually delete them. + +### Number of new and total features **(FREE SAAS)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5. + +On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project. + +![Feature count](img/feature_count_v14_6.png "Number of features in a release") + +The totals are displayed on [shields](https://shields.io/) and are generated per release by +[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake). + +| Item | Formula | +|------------------|------------------------------------------------------------------------------------| +| `New features` | Total count of release posts across all tiers for a single release in the project. | +| `Total features` | Total count of release posts in reverse order for all releases in the project. | + +The counts are also shown by license tier. + +| Item | Formula | +|------------------|-----------------------------------------------------------------------------------------------------| +| `New features` | Total count of release posts across a single tier for a single release in the project. | +| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. | diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index b79b94f4d54..61de003c28d 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -9,7 +9,7 @@ module Gitlab VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze def self.parse(str, parse_suffix: false) - if str.is_a?(self.class) + if str.is_a?(self) str elsif str && m = str.match(VERSION_REGEX) VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) diff --git a/spec/features/merge_request/user_opens_context_commits_modal_spec.rb b/spec/features/merge_request/user_opens_context_commits_modal_spec.rb new file mode 100644 index 00000000000..2d574e57fe9 --- /dev/null +++ b/spec/features/merge_request/user_opens_context_commits_modal_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Merge request > Context commits', :js do + let(:user) { create(:user) } + let(:project) { create(:project, :public, :repository) } + let(:merge_request) { create(:merge_request, source_project: project) } + + before do + project.add_developer(user) + + sign_in(user) + + visit commits_project_merge_request_path(project, merge_request) + + wait_for_requests + end + + it 'opens modal' do + click_button 'Add previously merged commits' + + expect(page).to have_selector('#add-review-item') + expect(page).to have_content('Add or remove previously merged commits') + end +end diff --git a/spec/frontend/__helpers__/matchers/index.js b/spec/frontend/__helpers__/matchers/index.js index 5da6676cdc1..73464135950 100644 --- a/spec/frontend/__helpers__/matchers/index.js +++ b/spec/frontend/__helpers__/matchers/index.js @@ -3,3 +3,4 @@ export * from './to_have_tracking_attributes'; export * from './to_match_interpolated_text'; export * from './to_validate_json_schema'; export * from './to_match_expected_for_markdown'; +export * from './to_equal_graphql_fixture'; diff --git a/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture.js b/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture.js new file mode 100644 index 00000000000..1a095de1deb --- /dev/null +++ b/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture.js @@ -0,0 +1,28 @@ +import { isEqual } from 'lodash'; +import { stripTypenames } from 'helpers/graphql_helpers'; + +export function toEqualGraphqlFixture(received, match) { + let clearReceived; + let clearMatch; + + try { + clearReceived = JSON.parse(JSON.stringify(received)); + clearMatch = stripTypenames(match); + } catch (e) { + return { message: () => 'The comparator value is not an object', pass: false }; + } + const pass = isEqual(clearReceived, clearMatch); + // console.log(this.utils); + const message = pass + ? () => ` + Expected to not be: ${this.utils.printExpected(clearMatch)} + Received: ${this.utils.printReceived(clearReceived)} + ` + : () => + ` + Expected to be: ${this.utils.printExpected(clearMatch)} + Received: ${this.utils.printReceived(clearReceived)} + `; + + return { actual: received, message, pass }; +} diff --git a/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture_spec.js b/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture_spec.js new file mode 100644 index 00000000000..00adc8cfc8d --- /dev/null +++ b/spec/frontend/__helpers__/matchers/to_equal_graphql_fixture_spec.js @@ -0,0 +1,58 @@ +describe('custom matcher toEqualGraphqlFixture', () => { + const key = 'value'; + const key2 = 'value2'; + + describe('positive assertion', () => { + it.each([ + [{}, {}], + [{ undef: undefined }, { undef: undefined }], + [{}, { __typename: 'MyType' }], + [{ key }, { key, __typename: 'MyType' }], + [ + { obj: { key } }, + { + obj: { + key, + __typename: 'MyNestedType', + }, + __typename: 'MyType', + }, + ], + [[{ key }], [{ key }]], + [ + [{ key }], + [ + { + key, + __typename: 'MyCollectionType', + }, + ], + ], + ])('%j equals %j', (received, match) => { + expect(received).toEqualGraphqlFixture(match); + }); + }); + + describe('negative assertion', () => { + it.each([ + [{ __typename: 'MyType' }, {}], + [{ key }, { key2, __typename: 'MyType' }], + [ + { key, key2 }, + { key2, __typename: 'MyType' }, + ], + [[{ key }, { key2 }], [{ key }]], + [ + [{ key, key2 }], + [ + { + key, + __typename: 'MyCollectionType', + }, + ], + ], + ])('%j does not equal %j', (received, match) => { + expect(received).not.toEqualGraphqlFixture(match); + }); + }); +}); diff --git a/spec/frontend/releases/__snapshots__/util_spec.js.snap b/spec/frontend/releases/__snapshots__/util_spec.js.snap index 0a796b0dbe6..2a12d945792 100644 --- a/spec/frontend/releases/__snapshots__/util_spec.js.snap +++ b/spec/frontend/releases/__snapshots__/util_spec.js.snap @@ -214,7 +214,6 @@ Object { "selfUrl": "http://localhost/releases-namespace/releases-project/-/releases/v1.1", }, "assets": Object { - "count": undefined, "links": Array [ Object { "directAssetPath": "/binaries/awesome-app-3", @@ -247,29 +246,22 @@ Object { ], "sources": Array [], }, - "author": undefined, "description": "Best. Release. **Ever.** :rocket:", "evidences": Array [], "milestones": Array [ Object { "id": "gid://gitlab/Milestone/123", "issueStats": Object {}, - "stats": undefined, "title": "12.3", - "webPath": undefined, - "webUrl": undefined, }, Object { "id": "gid://gitlab/Milestone/124", "issueStats": Object {}, - "stats": undefined, "title": "12.4", - "webPath": undefined, - "webUrl": undefined, }, ], "name": "The first release", - "releasedAt": 2018-12-10T00:00:00.000Z, + "releasedAt": "2018-12-10T00:00:00.000Z", "tagName": "v1.1", "tagPath": "/releases-namespace/releases-project/-/tags/v1.1", }, diff --git a/spec/frontend/releases/util_spec.js b/spec/frontend/releases/util_spec.js index 055c8e8b39f..dfea3fc0037 100644 --- a/spec/frontend/releases/util_spec.js +++ b/spec/frontend/releases/util_spec.js @@ -7,6 +7,7 @@ import { convertAllReleasesGraphQLResponse, convertOneReleaseGraphQLResponse, } from '~/releases/util'; +import { stripTypenames } from 'helpers/graphql_helpers'; describe('releases/util.js', () => { describe('convertGraphQLRelease', () => { @@ -136,7 +137,7 @@ describe('releases/util.js', () => { describe('convertOneReleaseForEditingGraphQLResponse', () => { it('matches snapshot', () => { expect( - convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse), + stripTypenames(convertOneReleaseGraphQLResponse(originalOneReleaseForEditingQueryResponse)), ).toMatchSnapshot(); }); }); diff --git a/spec/frontend/runner/admin_runner_edit/admin_runner_edit_app_spec.js b/spec/frontend/runner/admin_runner_edit/admin_runner_edit_app_spec.js index 8a34cb14d8b..361a9aabf33 100644 --- a/spec/frontend/runner/admin_runner_edit/admin_runner_edit_app_spec.js +++ b/spec/frontend/runner/admin_runner_edit/admin_runner_edit_app_spec.js @@ -87,10 +87,10 @@ describe('AdminRunnerEditApp', () => { await createComponentWithApollo(); expect(findRunnerUpdateForm().props()).toMatchObject({ - runner: mockRunner, loading: false, runnerPath: mockRunnerPath, }); + expect(findRunnerUpdateForm().props('runner')).toEqualGraphqlFixture(mockRunner); }); describe('When there is an error', () => { diff --git a/spec/frontend/runner/admin_runner_show/admin_runner_show_app_spec.js b/spec/frontend/runner/admin_runner_show/admin_runner_show_app_spec.js index 509681c5a77..1ae6eed806f 100644 --- a/spec/frontend/runner/admin_runner_show/admin_runner_show_app_spec.js +++ b/spec/frontend/runner/admin_runner_show/admin_runner_show_app_spec.js @@ -256,7 +256,7 @@ describe('AdminRunnerShowApp', () => { await createComponent({ stubs }); expect(findJobCountBadge().text()).toBe('3'); - expect(findRunnersJobs().props('runner')).toEqual({ ...mockRunner, ...runner }); + expect(findRunnersJobs().props('runner')).toEqualGraphqlFixture({ ...mockRunner, ...runner }); }); }); }); diff --git a/spec/frontend/runner/admin_runners/admin_runners_app_spec.js b/spec/frontend/runner/admin_runners/admin_runners_app_spec.js index bff1cb1164d..263204915ba 100644 --- a/spec/frontend/runner/admin_runners/admin_runners_app_spec.js +++ b/spec/frontend/runner/admin_runners/admin_runners_app_spec.js @@ -167,7 +167,7 @@ describe('AdminRunnersApp', () => { it('shows the runners list', async () => { await createComponent(); - expect(findRunnerList().props('runners')).toEqual(mockRunners); + expect(findRunnerList().props('runners')).toEqualGraphqlFixture(mockRunners); }); it('runner item links to the runner admin page', async () => { @@ -188,7 +188,7 @@ describe('AdminRunnersApp', () => { const runnerActions = wrapper.find('tr [data-testid="td-actions"]').find(RunnerActionsCell); const runner = mockRunners[0]; - expect(runnerActions.props()).toEqual({ + expect(runnerActions.props()).toEqualGraphqlFixture({ runner, editUrl: runner.editAdminUrl, }); diff --git a/spec/frontend/runner/components/runner_jobs_spec.js b/spec/frontend/runner/components/runner_jobs_spec.js index 20582aaaf40..8e3715e45d2 100644 --- a/spec/frontend/runner/components/runner_jobs_spec.js +++ b/spec/frontend/runner/components/runner_jobs_spec.js @@ -73,8 +73,7 @@ describe('RunnerJobs', () => { it('Shows jobs', () => { const jobs = findRunnerJobsTable().props('jobs'); - expect(jobs).toHaveLength(mockJobs.length); - expect(jobs[0]).toMatchObject(mockJobs[0]); + expect(jobs).toEqualGraphqlFixture(mockJobs); }); describe('When "Next" page is clicked', () => { diff --git a/spec/frontend/runner/group_runner_show/group_runner_show_app_spec.js b/spec/frontend/runner/group_runner_show/group_runner_show_app_spec.js index cee1d436942..5d3552af322 100644 --- a/spec/frontend/runner/group_runner_show/group_runner_show_app_spec.js +++ b/spec/frontend/runner/group_runner_show/group_runner_show_app_spec.js @@ -107,7 +107,7 @@ describe('GroupRunnerShowApp', () => { }); it('renders runner details component', () => { - expect(findRunnerDetails().props('runner')).toEqual(mockRunner); + expect(findRunnerDetails().props('runner')).toEqualGraphqlFixture(mockRunner); }); describe('when runner cannot be updated', () => { diff --git a/spec/frontend/runner/group_runners/group_runners_app_spec.js b/spec/frontend/runner/group_runners/group_runners_app_spec.js index 99b6273627c..3012240d8f4 100644 --- a/spec/frontend/runner/group_runners/group_runners_app_spec.js +++ b/spec/frontend/runner/group_runners/group_runners_app_spec.js @@ -167,7 +167,7 @@ describe('GroupRunnersApp', () => { await createComponent(); const runners = findRunnerList().props('runners'); - expect(runners).toEqual(mockGroupRunnersEdges.map(({ node }) => node)); + expect(runners).toEqualGraphqlFixture(mockGroupRunnersEdges.map(({ node }) => node)); }); it('requests the runners with group path and no other filters', async () => { diff --git a/spec/lib/gitlab/version_info_spec.rb b/spec/lib/gitlab/version_info_spec.rb index c7db222b9cd..078f952afad 100644 --- a/spec/lib/gitlab/version_info_spec.rb +++ b/spec/lib/gitlab/version_info_spec.rb @@ -84,6 +84,7 @@ RSpec.describe Gitlab::VersionInfo do end describe '.parse' do + it { expect(described_class.parse(described_class.new(1, 0, 0))).to eq(@v1_0_0) } it { expect(described_class.parse("1.0.0")).to eq(@v1_0_0) } it { expect(described_class.parse("1.0.0.1")).to eq(@v1_0_0) } it { expect(described_class.parse("1.0.0-ee")).to eq(@v1_0_0) } diff --git a/spec/support/helpers/javascript_fixtures_helpers.rb b/spec/support/helpers/javascript_fixtures_helpers.rb index 84cd0181533..32e6e8d50bd 100644 --- a/spec/support/helpers/javascript_fixtures_helpers.rb +++ b/spec/support/helpers/javascript_fixtures_helpers.rb @@ -2,6 +2,7 @@ require 'action_dispatch/testing/test_request' require 'fileutils' +require 'graphlyte' require_relative '../../../lib/gitlab/popen' @@ -47,7 +48,8 @@ module JavaScriptFixturesHelpers path = Rails.root / base / query_path queries = Gitlab::Graphql::Queries.find(path) if queries.length == 1 - queries.first.text(mode: Gitlab.ee? ? :ee : :ce ) + query = queries.first.text(mode: Gitlab.ee? ? :ee : :ce ) + inflate_query_with_typenames(query) else raise "Could not find query file at #{path}, please check your query_path" % path end @@ -55,6 +57,23 @@ module JavaScriptFixturesHelpers private + # Private: Parse a GraphQL query and inflate the fields with a __typename + # + # query - the GraqhQL query to parse + def inflate_query_with_typenames(query, doc: Graphlyte.parse(query)) + typename_editor.edit(doc) + + doc.to_s + end + + def typename_editor + typename = Graphlyte::Syntax::Field.new(name: '__typename') + + @editor ||= Graphlyte::Editor.new.on_field do |field| + field.selection << typename unless field.selection.empty? || field.selection.map(&:name).include?('__typename') + end + end + # Private: Store a response object as fixture file # # response - string or response object to store diff --git a/vendor/gems/mail-smtp_pool/.gitlab-ci.yml b/vendor/gems/mail-smtp_pool/.gitlab-ci.yml index 4c9e62f7d11..620be9e531c 100644 --- a/vendor/gems/mail-smtp_pool/.gitlab-ci.yml +++ b/vendor/gems/mail-smtp_pool/.gitlab-ci.yml @@ -17,10 +17,6 @@ workflow: script: - bundle exec rspec -rspec-2.6: - image: "ruby:2.6" - extends: .rspec - rspec-2.7: image: "ruby:2.7" extends: .rspec diff --git a/vendor/gems/omniauth-gitlab/.gitlab-ci.yml b/vendor/gems/omniauth-gitlab/.gitlab-ci.yml index ad9545e2998..b1a59768819 100644 --- a/vendor/gems/omniauth-gitlab/.gitlab-ci.yml +++ b/vendor/gems/omniauth-gitlab/.gitlab-ci.yml @@ -17,14 +17,10 @@ workflow: script: - bundle exec rspec -rspec-2.6: - image: "ruby:2.6" - extends: .rspec - rspec-2.7: image: "ruby:2.7" extends: .rspec rspec-3.0: image: "ruby:3.0" - extends: .rspec \ No newline at end of file + extends: .rspec diff --git a/vendor/gems/omniauth_crowd/.gitlab-ci.yml b/vendor/gems/omniauth_crowd/.gitlab-ci.yml index 98e686db4e3..750778d0ce6 100644 --- a/vendor/gems/omniauth_crowd/.gitlab-ci.yml +++ b/vendor/gems/omniauth_crowd/.gitlab-ci.yml @@ -17,10 +17,6 @@ workflow: script: - bundle exec rspec -rspec-2.6: - image: "ruby:2.6" - extends: .rspec - rspec-2.7: image: "ruby:2.7" extends: .rspec diff --git a/vendor/gems/omniauth_crowd/Gemfile.lock b/vendor/gems/omniauth_crowd/Gemfile.lock index 0ac781e9948..56c9bd4cc7e 100644 --- a/vendor/gems/omniauth_crowd/Gemfile.lock +++ b/vendor/gems/omniauth_crowd/Gemfile.lock @@ -9,49 +9,51 @@ PATH GEM remote: http://rubygems.org/ specs: - activesupport (5.0.0.1) + activesupport (7.0.3.1) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (~> 0.7) - minitest (~> 5.1) - tzinfo (~> 1.1) - addressable (2.5.2) - public_suffix (>= 2.0.2, < 4.0) - concurrent-ruby (1.0.5) - crack (0.4.3) - safe_yaml (~> 1.0.0) - diff-lcs (1.2.5) - hashdiff (0.3.6) - hashie (3.4.3) - i18n (0.8.1) - mini_portile2 (2.1.0) - minitest (5.10.1) - nokogiri (1.6.8.1) - mini_portile2 (~> 2.1.0) - omniauth (1.3.1) - hashie (>= 1.2, < 4) - rack (>= 1.0, < 3) - public_suffix (3.0.0) - rack (1.6.4) - rack-test (0.6.3) - rack (>= 1.0) - rake (10.5.0) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.0) + public_suffix (>= 2.0.2, < 5.0) + concurrent-ruby (1.1.10) + crack (0.4.5) + rexml + diff-lcs (1.5.0) + hashdiff (1.0.1) + hashie (5.0.0) + i18n (1.12.0) + concurrent-ruby (~> 1.0) + mini_portile2 (2.8.0) + minitest (5.16.2) + nokogiri (1.13.8) + mini_portile2 (~> 2.8.0) + racc (~> 1.4) + omniauth (1.9.1) + hashie (>= 3.4.6) + rack (>= 1.6.2, < 3) + public_suffix (4.0.7) + racc (1.6.0) + rack (2.2.4) + rack-test (2.0.2) + rack (>= 1.3) + rake (13.0.6) rexml (3.2.5) - rspec (3.0.0) - rspec-core (~> 3.0.0) - rspec-expectations (~> 3.0.0) - rspec-mocks (~> 3.0.0) - rspec-core (3.0.4) - rspec-support (~> 3.0.0) - rspec-expectations (3.0.4) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.0.0) - rspec-mocks (3.0.4) - rspec-support (~> 3.0.0) - rspec-support (3.0.4) - safe_yaml (1.0.4) - thread_safe (0.3.6) - tzinfo (1.2.2) - thread_safe (~> 0.1) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-support (3.11.0) + tzinfo (2.0.5) + concurrent-ruby (~> 1.0) webmock (3.0.1) addressable (>= 2.3.6) crack (>= 0.3.2) @@ -67,8 +69,8 @@ DEPENDENCIES rack-test rake rexml (~> 3.2.5) - rspec (~> 3.0.0) + rspec (>= 3.4) webmock (~> 3.0.0) BUNDLED WITH - 2.3.15 + 2.3.19 diff --git a/vendor/gems/omniauth_crowd/omniauth_crowd.gemspec b/vendor/gems/omniauth_crowd/omniauth_crowd.gemspec index 4963c5fef1b..1707c7f3f10 100644 --- a/vendor/gems/omniauth_crowd/omniauth_crowd.gemspec +++ b/vendor/gems/omniauth_crowd/omniauth_crowd.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |gem| gem.add_development_dependency(%q, [">= 0"]) gem.add_development_dependency(%q, [">= 0"]) gem.add_development_dependency(%q, ["~> 3.2.5"]) - gem.add_development_dependency(%q, ["~> 3.0.0"]) + gem.add_development_dependency(%q, [">= 3.4"]) gem.add_development_dependency(%q, ["~> 3.0.0"]) gem.add_development_dependency(%q, ["> 1.0.0"]) end