diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index c0619e6bf3f..ad340ae0947 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -020b5f709d58277c360ba409b8f8a9e81cee2781 +fa974a4ab21aa6acc4c3a00456265248a4d70703 diff --git a/app/graphql/types/issue_type.rb b/app/graphql/types/issue_type.rb index 6bf7d812420..acc7b97d42e 100644 --- a/app/graphql/types/issue_type.rb +++ b/app/graphql/types/issue_type.rb @@ -62,6 +62,8 @@ module Types description: 'Number of downvotes the issue has received' field :user_notes_count, GraphQL::INT_TYPE, null: false, description: 'Number of user notes of the issue' + field :user_discussions_count, GraphQL::INT_TYPE, null: false, + description: 'Number of user discussions in the issue' field :web_path, GraphQL::STRING_TYPE, null: false, method: :issue_path, description: 'Web path of the issue' field :web_url, GraphQL::STRING_TYPE, null: false, @@ -113,6 +115,26 @@ module Types field :severity, Types::IssuableSeverityEnum, null: true, description: 'Severity level of the incident' + def user_notes_count + BatchLoader::GraphQL.for(object.id).batch(key: :issue_user_notes_count) do |ids, loader, args| + counts = Note.count_for_collection(ids, 'Issue').index_by(&:noteable_id) + + ids.each do |id| + loader.call(id, counts[id]&.count || 0) + end + end + end + + def user_discussions_count + BatchLoader::GraphQL.for(object.id).batch(key: :issue_user_discussions_count) do |ids, loader, args| + counts = Note.count_for_collection(ids, 'Issue', 'COUNT(DISTINCT discussion_id) as count').index_by(&:noteable_id) + + ids.each do |id| + loader.call(id, counts[id]&.count || 0) + end + end + end + def author Gitlab::Graphql::Loaders::BatchModelLoader.new(User, object.author_id).find end diff --git a/app/graphql/types/merge_request_type.rb b/app/graphql/types/merge_request_type.rb index c35316fe374..acf657ca375 100644 --- a/app/graphql/types/merge_request_type.rb +++ b/app/graphql/types/merge_request_type.rb @@ -68,6 +68,8 @@ module Types description: 'SHA of the merge request commit (set once merged)' field :user_notes_count, GraphQL::INT_TYPE, null: true, description: 'User notes count of the merge request' + field :user_discussions_count, GraphQL::INT_TYPE, null: true, + description: 'Number of user discussions in the merge request' field :should_remove_source_branch, GraphQL::BOOLEAN_TYPE, method: :should_remove_source_branch?, null: true, description: 'Indicates if the source branch of the merge request will be deleted after merge' field :force_remove_source_branch, GraphQL::BOOLEAN_TYPE, method: :force_remove_source_branch?, null: true, @@ -158,17 +160,25 @@ module Types object.approved_by_users end - # rubocop: disable CodeReuse/ActiveRecord def user_notes_count BatchLoader::GraphQL.for(object.id).batch(key: :merge_request_user_notes_count) do |ids, loader, args| - counts = Note.where(noteable_type: 'MergeRequest', noteable_id: ids).user.group(:noteable_id).count + counts = Note.count_for_collection(ids, 'MergeRequest').index_by(&:noteable_id) ids.each do |id| - loader.call(id, counts[id] || 0) + loader.call(id, counts[id]&.count || 0) + end + end + end + + def user_discussions_count + BatchLoader::GraphQL.for(object.id).batch(key: :merge_request_user_discussions_count) do |ids, loader, args| + counts = Note.count_for_collection(ids, 'MergeRequest', 'COUNT(DISTINCT discussion_id) as count').index_by(&:noteable_id) + + ids.each do |id| + loader.call(id, counts[id]&.count || 0) end end end - # rubocop: enable CodeReuse/ActiveRecord def diff_stats(path: nil) stats = Array.wrap(object.diff_stats&.to_a) diff --git a/app/models/note.rb b/app/models/note.rb index 954843505d4..cfdac6c432f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -197,8 +197,8 @@ class Note < ApplicationRecord .map(&:position) end - def count_for_collection(ids, type) - user.select('noteable_id', 'COUNT(*) as count') + def count_for_collection(ids, type, count_column = 'COUNT(*) as count') + user.select(:noteable_id, count_column) .group(:noteable_id) .where(noteable_type: type, noteable_id: ids) end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 3996965b54a..9a4562793d7 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -10,7 +10,9 @@ = notice[:message].html_safe - if @license.present? && show_license_breakdown? - = render_if_exists 'admin/licenses/breakdown' + .license-panel.gl-mt-5 + = render_if_exists 'admin/licenses/summary' + = render_if_exists 'admin/licenses/breakdown' .admin-dashboard.gl-mt-3 .row diff --git a/app/views/shared/milestones/_delete_button.html.haml b/app/views/shared/milestones/_delete_button.html.haml index 7a813e110c4..09c783a0b24 100644 --- a/app/views/shared/milestones/_delete_button.html.haml +++ b/app/views/shared/milestones/_delete_button.html.haml @@ -1,6 +1,6 @@ - milestone_url = @milestone.project_milestone? ? project_milestone_path(@project, @milestone) : group_milestone_path(@group, @milestone) -%button.js-delete-milestone-button.btn.btn-grouped.btn-danger{ data: { milestone_id: @milestone.id, +%button.js-delete-milestone-button.btn.gl-button.btn-grouped.btn-danger{ data: { milestone_id: @milestone.id, milestone_title: markdown_field(@milestone, :title), milestone_url: milestone_url, milestone_issue_count: @milestone.issues.count, diff --git a/app/views/shared/milestones/_header.html.haml b/app/views/shared/milestones/_header.html.haml index ea90b674b34..93da319fce7 100644 --- a/app/views/shared/milestones/_header.html.haml +++ b/app/views/shared/milestones/_header.html.haml @@ -11,10 +11,10 @@ .milestone-buttons - if can?(current_user, :admin_milestone, @group || @project) - = link_to _('Edit'), edit_milestone_path(milestone), class: 'btn btn-grouped' + = link_to _('Edit'), edit_milestone_path(milestone), class: 'btn gl-button btn-grouped' - if milestone.project_milestone? && milestone.project.group - %button.js-promote-project-milestone-button.btn.btn-grouped{ data: { toggle: 'modal', + %button.js-promote-project-milestone-button.btn.gl-button.btn-grouped{ data: { toggle: 'modal', target: '#promote-milestone-modal', milestone_title: milestone.title, group_name: milestone.project.group.name, @@ -26,11 +26,11 @@ #promote-milestone-modal - if milestone.active? - = link_to _('Close milestone'), update_milestone_path(milestone, { state_event: :close }), method: :put, class: 'btn btn-grouped btn-close' + = link_to _('Close milestone'), update_milestone_path(milestone, { state_event: :close }), method: :put, class: 'btn gl-button btn-grouped btn-close' - else - = link_to _('Reopen milestone'), update_milestone_path(milestone, { state_event: :activate }), method: :put, class: 'btn btn-grouped btn-reopen' + = link_to _('Reopen milestone'), update_milestone_path(milestone, { state_event: :activate }), method: :put, class: 'btn gl-button btn-grouped btn-reopen' = render 'shared/milestones/delete_button' - %button.btn.btn-default.btn-grouped.float-right.d-block.d-sm-none.js-sidebar-toggle{ type: 'button' } + %button.btn.gl-button.btn-default.btn-grouped.float-right.d-block.d-sm-none.js-sidebar-toggle{ type: 'button' } = sprite_icon('chevron-double-lg-left') diff --git a/app/views/shared/milestones/_labels_tab.html.haml b/app/views/shared/milestones/_labels_tab.html.haml index 3b4d29ca7b0..5202167e87a 100644 --- a/app/views/shared/milestones/_labels_tab.html.haml +++ b/app/views/shared/milestones/_labels_tab.html.haml @@ -8,7 +8,7 @@ = markdown_field(label, :description) .float-right.d-none.d-lg-block.d-xl-block - = link_to milestones_issues_path(options.merge(state: 'opened')), class: 'btn btn-transparent btn-action' do + = link_to milestones_issues_path(options.merge(state: 'opened')), class: 'btn gl-button btn-default-tertiary btn-action' do - pluralize milestone_issues_by_label_count(@milestone, label, state: :opened), _('open issue') - = link_to milestones_issues_path(options.merge(state: 'closed')), class: 'btn btn-transparent btn-action' do + = link_to milestones_issues_path(options.merge(state: 'closed')), class: 'btn gl-button btn-default-tertiary btn-action' do - pluralize milestone_issues_by_label_count(@milestone, label, state: :closed), _('closed issue') diff --git a/app/views/shared/milestones/_milestone.html.haml b/app/views/shared/milestones/_milestone.html.haml index f28aa406784..1597a011a45 100644 --- a/app/views/shared/milestones/_milestone.html.haml +++ b/app/views/shared/milestones/_milestone.html.haml @@ -46,7 +46,7 @@ .milestone-actions.d-flex.justify-content-sm-start.justify-content-md-end - if @project # if in milestones list on project level - if can_admin_group_milestones? - %button.js-promote-project-milestone-button.btn.btn-blank.btn-sm.btn-grouped.has-tooltip{ title: s_('Milestones|Promote to Group Milestone'), + %button.js-promote-project-milestone-button.btn.gl-button.btn-default-tertiary.btn-sm.btn-grouped.has-tooltip{ title: s_('Milestones|Promote to Group Milestone'), disabled: true, type: 'button', data: { url: promote_project_milestone_path(milestone.project, milestone), @@ -59,6 +59,6 @@ - if can?(current_user, :admin_milestone, milestone) - if milestone.closed? - = link_to s_('Milestones|Reopen Milestone'), milestone_path(milestone, milestone: { state_event: :activate }), method: :put, class: "btn btn-sm btn-grouped btn-reopen" + = link_to s_('Milestones|Reopen Milestone'), milestone_path(milestone, milestone: { state_event: :activate }), method: :put, class: "btn gl-button btn-sm btn-grouped btn-reopen" - else - = link_to s_('Milestones|Close Milestone'), milestone_path(milestone, milestone: { state_event: :close }), method: :put, class: "btn btn-sm btn-grouped btn-close" + = link_to s_('Milestones|Close Milestone'), milestone_path(milestone, milestone: { state_event: :close }), method: :put, class: "btn gl-button btn-warning-secondary btn-sm btn-grouped btn-close" diff --git a/changelogs/unreleased/262112_populate_missing_dismissal_information_for_vulnerabilities.yml b/changelogs/unreleased/262112_populate_missing_dismissal_information_for_vulnerabilities.yml new file mode 100644 index 00000000000..44a62049cfd --- /dev/null +++ b/changelogs/unreleased/262112_populate_missing_dismissal_information_for_vulnerabilities.yml @@ -0,0 +1,5 @@ +--- +title: Populate missing `dismissed_at` and `dismissed_by_id` attributes of vulnerabilities +merge_request: 46370 +author: +type: fixed diff --git a/changelogs/unreleased/add-user-discussions-count-to-issues-and-merge-requests.yml b/changelogs/unreleased/add-user-discussions-count-to-issues-and-merge-requests.yml new file mode 100644 index 00000000000..1c8dc95d6fe --- /dev/null +++ b/changelogs/unreleased/add-user-discussions-count-to-issues-and-merge-requests.yml @@ -0,0 +1,5 @@ +--- +title: Add userDiscussionsCount to issues and merge requests GraphQL +merge_request: 46311 +author: +type: added diff --git a/config/feature_flags/development/ci_include_multiple_files_from_project.yml b/config/feature_flags/development/ci_include_multiple_files_from_project.yml new file mode 100644 index 00000000000..45cfe47cf51 --- /dev/null +++ b/config/feature_flags/development/ci_include_multiple_files_from_project.yml @@ -0,0 +1,7 @@ +--- +name: ci_include_multiple_files_from_project +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45991 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/271560 +type: development +group: group::pipeline authoring +default_enabled: false diff --git a/config/initializers/console_message.rb b/config/initializers/console_message.rb index af9064250d2..fe47195062b 100644 --- a/config/initializers/console_message.rb +++ b/config/initializers/console_message.rb @@ -20,4 +20,15 @@ if Gitlab::Runtime.console? end puts '-' * 80 + + # Stop irb from writing a history file by default. + module IrbNoHistory + def init_config(*) + super + + IRB.conf[:SAVE_HISTORY] = false + end + end + + IRB.singleton_class.prepend(IrbNoHistory) end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index ded8b769ff1..2804322e029 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -144,6 +144,8 @@ - 1 - - group_import - 1 +- - group_saml_group_sync + - 1 - - hashed_storage - 1 - - import_issues_csv diff --git a/db/migrate/20201028160831_add_temporary_index_to_vulnerabilities_table.rb b/db/migrate/20201028160831_add_temporary_index_to_vulnerabilities_table.rb new file mode 100644 index 00000000000..05bb75be75a --- /dev/null +++ b/db/migrate/20201028160831_add_temporary_index_to_vulnerabilities_table.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddTemporaryIndexToVulnerabilitiesTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'temporary_index_vulnerabilities_on_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :vulnerabilities, :id, where: "state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :vulnerabilities, INDEX_NAME + end +end diff --git a/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb b/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb new file mode 100644 index 00000000000..f358ea863db --- /dev/null +++ b/db/post_migrate/20201028160832_schedule_populate_missing_dismissal_information_for_vulnerabilities.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class SchedulePopulateMissingDismissalInformationForVulnerabilities < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 1_000 + DELAY_INTERVAL = 3.minutes.to_i + MIGRATION_CLASS = 'PopulateMissingVulnerabilityDismissalInformation' + + disable_ddl_transaction! + + def up + ::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Vulnerability.broken.each_batch(of: BATCH_SIZE) do |batch, index| + vulnerability_ids = batch.pluck(:id) + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, vulnerability_ids) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb b/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb new file mode 100644 index 00000000000..64d22863389 --- /dev/null +++ b/db/post_migrate/20201029052241_migrate_geo_blob_verification_primary_worker_sidekiq_queue.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class MigrateGeoBlobVerificationPrimaryWorkerSidekiqQueue < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + sidekiq_queue_migrate 'geo:geo_blob_verification_primary', to: 'geo:geo_verification' + end + + def down + sidekiq_queue_migrate 'geo:geo_verification', to: 'geo:geo_blob_verification_primary' + end +end diff --git a/db/schema_migrations/20201028160831 b/db/schema_migrations/20201028160831 new file mode 100644 index 00000000000..de94901dcc5 --- /dev/null +++ b/db/schema_migrations/20201028160831 @@ -0,0 +1 @@ +4b0c70d8cd2648149011adab4f302922483436406f361c3037f26efb12b19042 \ No newline at end of file diff --git a/db/schema_migrations/20201028160832 b/db/schema_migrations/20201028160832 new file mode 100644 index 00000000000..4b0da32b638 --- /dev/null +++ b/db/schema_migrations/20201028160832 @@ -0,0 +1 @@ +9ea8e8f1234d6291ea00e725d380bfe33d804853b90da1221be8781b3dd9bb77 \ No newline at end of file diff --git a/db/schema_migrations/20201029052241 b/db/schema_migrations/20201029052241 new file mode 100644 index 00000000000..8257e4d3681 --- /dev/null +++ b/db/schema_migrations/20201029052241 @@ -0,0 +1 @@ +87e330bc15accb10733825b079cf89e78d905a7c4080075489857085f014bfe7 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index f36f0b7de77..d955a3e6dcd 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -22200,6 +22200,8 @@ CREATE UNIQUE INDEX snippet_user_mentions_on_snippet_id_index ON snippet_user_me CREATE UNIQUE INDEX taggings_idx ON taggings USING btree (tag_id, taggable_id, taggable_type, context, tagger_id, tagger_type); +CREATE INDEX temporary_index_vulnerabilities_on_id ON vulnerabilities USING btree (id) WHERE ((state = 2) AND ((dismissed_at IS NULL) OR (dismissed_by_id IS NULL))); + CREATE UNIQUE INDEX term_agreements_unique_index ON term_agreements USING btree (user_id, term_id); CREATE INDEX terraform_state_versions_verification_checksum_partial ON terraform_state_versions USING btree (verification_checksum) WHERE (verification_checksum IS NOT NULL); diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index f375f3287d0..5bdabe547c4 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -133,9 +133,10 @@ Note the following when promoting a secondary: ``` 1. Promote the **secondary** node to the **primary** node. - -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. + CAUTION: **Caution:** + If the secondary node [has been paused](../../geo/index.md#pausing-and-resuming-replication), this performs + a point-in-time recovery to the last known state. + Data that was created on the primary while the secondary was paused will be lost. To promote the secondary node to primary along with preflight checks: @@ -166,14 +167,16 @@ conjunction with multiple servers, as it can only perform changes on a **secondary** with only a single machine. Instead, you must do this manually. -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. +CAUTION: **Caution:** + If the secondary node [has been paused](../../geo/index.md#pausing-and-resuming-replication), this performs +a point-in-time recovery to the last known state. +Data that was created on the primary while the secondary was paused will be lost. 1. SSH in to the database node in the **secondary** and trigger PostgreSQL to promote to read-write: ```shell - sudo gitlab-pg-ctl promote + sudo gitlab-ctl promote-db ``` In GitLab 12.8 and earlier, see [Message: `sudo: gitlab-pg-ctl: command not found`](../replication/troubleshooting.md#message-sudo-gitlab-pg-ctl-command-not-found). @@ -211,9 +214,6 @@ an external PostgreSQL database, as it can only perform changes on a **secondary node with GitLab and the database on the same machine. As a result, a manual process is required: -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. - 1. Promote the replica database associated with the **secondary** site. This will set the database to read-write: - Amazon RDS - [Promoting a Read Replica to Be a Standalone DB Instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 7062a30787c..c89b7929b13 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -227,14 +227,15 @@ conjunction with multiple servers, as it can only perform changes on a **secondary** with only a single machine. Instead, you must do this manually. -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. +CAUTION: **Caution:** + If the secondary node [has been paused](../../../geo/index.md#pausing-and-resuming-replication), this performs +a point-in-time recovery to the last known state. +Data that was created on the primary while the secondary was paused will be lost. -1. SSH in to the PostgreSQL node in the **secondary** and trigger PostgreSQL to - promote to read-write: +1. SSH in to the PostgreSQL node in the **secondary** and promote PostgreSQL separately: ```shell - sudo gitlab-pg-ctl promote + sudo gitlab-ctl promote-db ``` In GitLab 12.8 and earlier, see [Message: `sudo: gitlab-pg-ctl: command not found`](../../replication/troubleshooting.md#message-sudo-gitlab-pg-ctl-command-not-found). diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 10e5eb8cc61..085a2b52674 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -196,8 +196,9 @@ For information on how to update your Geo nodes to the latest GitLab version, se > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35913) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2. -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. +CAUTION: **Caution:** +Pausing and resuming of replication is currently only supported for Geo installations using an +Omnibus GitLab-managed database. External databases are currently not supported. In some circumstances, like during [upgrades](replication/updating_the_geo_nodes.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index a5d35816fd1..55ddccb1d98 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -21,9 +21,6 @@ Updating Geo nodes involves performing: NOTE: **Note:** These general update steps are not intended for [high-availability deployments](https://docs.gitlab.com/omnibus/update/README.html#multi-node--ha-deployment), and will cause downtime. If you want to avoid downtime, consider using [zero downtime updates](https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates). -DANGER: **Warning:** -In GitLab 13.2 and later versions, promoting a secondary node to a primary while the secondary is paused fails. We are [investigating the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/225173). Do not pause replication before promoting a secondary. If the node is paused, please resume before promoting. - To update the Geo nodes when a new GitLab version is released, update **primary** and all **secondary** nodes: diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 3cce32ea8fd..e39c8919706 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -7495,6 +7495,11 @@ type EpicIssue implements CurrentUserTodos & Noteable { """ upvotes: Int! + """ + Number of user discussions in the issue + """ + userDiscussionsCount: Int! + """ Number of user notes of the issue """ @@ -9959,6 +9964,11 @@ type Issue implements CurrentUserTodos & Noteable { """ upvotes: Int! + """ + Number of user discussions in the issue + """ + userDiscussionsCount: Int! + """ Number of user notes of the issue """ @@ -12000,6 +12010,11 @@ type MergeRequest implements CurrentUserTodos & Noteable { """ upvotes: Int! + """ + Number of user discussions in the merge request + """ + userDiscussionsCount: Int + """ User notes count of the merge request """ diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 3b5a30c0bd0..1e5ac96f3ea 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -20682,6 +20682,24 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "userDiscussionsCount", + "description": "Number of user discussions in the issue", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "userNotesCount", "description": "Number of user notes of the issue", @@ -27153,6 +27171,24 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "userDiscussionsCount", + "description": "Number of user discussions in the issue", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "userNotesCount", "description": "Number of user notes of the issue", @@ -32838,6 +32874,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "userDiscussionsCount", + "description": "Number of user discussions in the merge request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "userNotesCount", "description": "User notes count of the merge request", diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 2db86a3ccd7..951736f1b5c 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -1191,6 +1191,7 @@ Relationship between an epic and an issue. | `updatedAt` | Time! | Timestamp of when the issue was last updated | | `updatedBy` | User | User that last updated the issue | | `upvotes` | Int! | Number of upvotes the issue has received | +| `userDiscussionsCount` | Int! | Number of user discussions in the issue | | `userNotesCount` | Int! | Number of user notes of the issue | | `userPermissions` | IssuePermissions! | Permissions for the current user on the resource | | `webPath` | String! | Web path of the issue | @@ -1426,6 +1427,7 @@ Represents a recorded measurement (object count) for the Admins. | `updatedAt` | Time! | Timestamp of when the issue was last updated | | `updatedBy` | User | User that last updated the issue | | `upvotes` | Int! | Number of upvotes the issue has received | +| `userDiscussionsCount` | Int! | Number of user discussions in the issue | | `userNotesCount` | Int! | Number of user notes of the issue | | `userPermissions` | IssuePermissions! | Permissions for the current user on the resource | | `webPath` | String! | Web path of the issue | @@ -1728,6 +1730,7 @@ Autogenerated return type of MarkAsSpamSnippet. | `totalTimeSpent` | Int! | Total time reported as spent on the merge request | | `updatedAt` | Time! | Timestamp of when the merge request was last updated | | `upvotes` | Int! | Number of upvotes for the merge request | +| `userDiscussionsCount` | Int | Number of user discussions in the merge request | | `userNotesCount` | Int | User notes count of the merge request | | `userPermissions` | MergeRequestPermissions! | Permissions for the current user on the resource | | `webUrl` | String | Web URL of the merge request | diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index d1ca3431826..889dd9c4266 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -320,6 +320,46 @@ services: command: ["--registry-mirror", "https://registry-mirror.example.com"] # Specify the registry mirror to use. ``` +#### DinD service defined inside of GitLab Runner configuration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27173) in GitLab Runner 13.6. + +If you are an administrator of GitLab Runner and you have the `dind` +service defined for the [Docker +executor](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersdockerservices-section), +or the [Kubernetes +executor](https://docs.gitlab.com/runner/executors/kubernetes.html#using-services) +you can specify the `command` to configure the registry mirror for the +Docker daemon. + +Docker: + +```toml +[[runners]] + ... + executor = "docker" + [runners.docker] + ... + privileged = true + [[runners.docker.services]] + name = "docker:19.03.13-dind" + command = ["--registry-mirror", "https://registry-mirror.example.com"] +``` + +Kubernetes: + +```toml +[[runners]] + ... + name = "kubernetes" + [runners.kubernetes] + ... + privileged = true + [[runners.kubernetes.services]] + name = "docker:19.03.13-dind" + command = ["--registry-mirror", "https://registry-mirror.example.com"] +``` + ##### Docker executor inside GitLab Runner configuration If you are an administrator of GitLab Runner and you always want to use diff --git a/doc/ci/environments/img/protected_access_group_v13_6.png b/doc/ci/environments/img/protected_access_group_v13_6.png index a67a332b7e5..9c39e4362e8 100644 Binary files a/doc/ci/environments/img/protected_access_group_v13_6.png and b/doc/ci/environments/img/protected_access_group_v13_6.png differ diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 980cb3d6703..4aea6142801 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -438,6 +438,42 @@ All [nested includes](#nested-includes) are executed in the scope of the target This means you can use local (relative to target project), project, remote, or template includes. +##### Multiple files from a project + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26793) in GitLab 13.6. +> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. **(CORE ONLY)** + +You can include multiple files from the same project: + +```yaml +include: + - project: 'my-group/my-project' + ref: master + file: + - '/templates/.builds.yml' + - '/templates/.tests.yml' +``` + +Including multiple files from the same project is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:ci_include_multiple_files_from_project) +``` + +To disable it: + +```ruby +Feature.disable(:ci_include_multiple_files_from_project) +``` + #### `include:remote` `include:remote` can be used to include a file from a different location, diff --git a/doc/development/fe_guide/img/editor_lite_create_ext.png b/doc/development/fe_guide/img/editor_lite_create_ext.png index d341eec3e96..87aac325593 100644 Binary files a/doc/development/fe_guide/img/editor_lite_create_ext.png and b/doc/development/fe_guide/img/editor_lite_create_ext.png differ diff --git a/doc/development/fe_guide/img/editor_lite_loading.png b/doc/development/fe_guide/img/editor_lite_loading.png index 183a9b02429..f2c242da155 100644 Binary files a/doc/development/fe_guide/img/editor_lite_loading.png and b/doc/development/fe_guide/img/editor_lite_loading.png differ diff --git a/doc/development/img/architecture_simplified.png b/doc/development/img/architecture_simplified.png index 72d00b91129..bd731758ddd 100644 Binary files a/doc/development/img/architecture_simplified.png and b/doc/development/img/architecture_simplified.png differ diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index a69494fcf60..b7ab009a1ed 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -125,10 +125,10 @@ read this section on [how to prepare the merge request for a database review](da ## Query Counts -**Summary:** a merge request **should not** increase the number of executed SQL +**Summary:** a merge request **should not** increase the total number of executed SQL queries unless absolutely necessary. -The number of queries executed by the code modified or added by a merge request +The total number of queries executed by the code modified or added by a merge request must not increase unless absolutely necessary. When building features it's entirely possible you will need some extra queries, but you should try to keep this at a minimum. @@ -147,7 +147,7 @@ end This will end up running one query for every object to update. This code can easily overload a database given enough rows to update or many instances of this code running in parallel. This particular problem is known as the -["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecoder](query_recorder.md) to detect this and prevent regressions. +["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecorder](query_recorder.md) to detect this and prevent regressions. In this particular case the workaround is fairly easy: @@ -158,6 +158,82 @@ objects_to_update.update_all(some_field: some_value) This uses ActiveRecord's `update_all` method to update all rows in a single query. This in turn makes it much harder for this code to overload a database. +## Cached Queries + +**Summary:** a merge request **should not** execute duplicated cached queries. + +Rails provides an [SQL query cache](https://guides.rubyonrails.org/caching_with_rails.html#sql-caching), +used to cache the results of database queries for the duration of the request. +If Rails encounters the same query again for that request, +it will use the cached result set as opposed to running the query against the database again. +The query results are only cached for the duration of that single request, it does not persist across multiple requests. + +The cached queries help with reducing DB load, but they still: + +- Consume memory. +- Require as to re-instantiate each `ActiveRecord` object. +- Require as to re-instantiate each relation of the object. +- Make us spend additional CPU-cycles to look into a list of cached queries. + +They are cheaper, but they are not cheap at all from `memory` perspective. + +Cached SQL queries, could mask [N+1 query problem](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). +If those N queries are executing the same query, it will not hit the database N times, it will return the cached results instead, +which is still expensive since we need to re-initialize objects each time, and this is CPU/Memory expensive. +Instead, you should use the same in-memory objects, if possible. + +When building features, you could use [Performance bar](../administration/monitoring/performance/performance_bar.md) +in order to list Database queries, which will include cached queries as well. If you see a lot of similar queries, +this often indicates an N+1 query issue (or a similar kind of query batching problem). +If you see same cached query executed multiple times, this often indicates a masked N+1 query problem. + +The code introduced by a merge request, should not execute multiple duplicated cached queries. + +The total number of the queries (including cached ones) executed by the code modified or added by a merge request +should not increase unless absolutely necessary. +The number of executed queries (including cached queries) should not depend on +collection size. +You can write a test by passing the `skip_cached` variable to [QueryRecorder](query_recorder.md) to detect this and prevent regressions. + +As an example, say you have a CI pipeline. All pipeline builds belong to the same pipeline, +thus they also belong to the same project (`pipeline.project`): + +```ruby +pipeline_project = pipeline.project +# Project Load (0.6ms) SELECT "projects".* FROM "projects" WHERE "projects"."id" = $1 LIMIT $2 +build = pipeline.builds.first + +build.project == pipeline_project +# CACHE Project Load (0.0ms) SELECT "projects".* FROM "projects" WHERE "projects"."id" = $1 LIMIT $2 +# => true +``` + +When we call `build.project`, it will not hit the database, it will use the cached result, but it will re-instantiate +same pipeline project object. It turns out that associated objects do not point to the same in-memory object. + +If we try to serialize each build: + +```ruby +pipeline.builds.each do |build| + build.to_json(only: [:name], include: [project: { only: [:name]}]) +end +``` + +It will re-instantiate project object for each build, instead of using the same in-memory object. + +In this particular case the workaround is fairly easy: + +```ruby +pipeline.builds.each do |build| + build.project = pipeline.project + build.to_json(only: [:name], include: [project: { only: [:name]}]) +end +``` + +We can assign `pipeline.project` to each `build.project`, since we know it should point to the same project. +This will allow us that each build point to the same in-memory project, +avoiding the cached SQL query and re-instantiation of the project object for each build. + ## Executing Queries in Loops **Summary:** SQL queries **must not** be executed in a loop unless absolutely diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index fb87e5137e5..d8951c64071 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -30,12 +30,13 @@ In some cases the query count might change slightly between runs for unrelated r ## Cached queries -By default, QueryRecorder will ignore cached queries in the count. However, it may be better to count -all queries to avoid introducing an N+1 query that may be masked by the statement cache. To do this, -pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher: +By default, QueryRecorder will ignore [cached queries](merge_request_performance_guidelines.md#cached-queries) in the count. However, it may be better to count +all queries to avoid introducing an N+1 query that may be masked by the statement cache. +To do this, this requires the `:use_sql_query_cache` flag to be set. +You should pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher: ```ruby -it "avoids N+1 database queries" do +it "avoids N+1 database queries", :use_sql_query_cache do control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page }.count create_list(:issue, 5) expect { visit_some_page }.not_to exceed_all_query_limit(control_count) @@ -123,4 +124,5 @@ There are multiple ways to find the source of queries. - [Bullet](profiling.md#bullet) For finding `N+1` query problems - [Performance guidelines](performance.md) -- [Merge request performance guidelines](merge_request_performance_guidelines.md#query-counts) +- [Merge request performance guidelines - Query counts](merge_request_performance_guidelines.md#query-counts) +- [Merge request performance guidelines - Cached queries](merge_request_performance_guidelines.md#cached-queries) diff --git a/doc/user/profile/account/img/register_v13_6.png b/doc/user/profile/account/img/register_v13_6.png index 7c6eb04de79..ce4adc0f55b 100644 Binary files a/doc/user/profile/account/img/register_v13_6.png and b/doc/user/profile/account/img/register_v13_6.png differ diff --git a/doc/user/project/img/group_issue_board.png b/doc/user/project/img/group_issue_board.png deleted file mode 100644 index be360d18540..00000000000 Binary files a/doc/user/project/img/group_issue_board.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_add_list.png b/doc/user/project/img/issue_board_add_list.png deleted file mode 100644 index 91098daa1d1..00000000000 Binary files a/doc/user/project/img/issue_board_add_list.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_add_list_v13_6.png b/doc/user/project/img/issue_board_add_list_v13_6.png new file mode 100644 index 00000000000..4239ab6e7e4 Binary files /dev/null and b/doc/user/project/img/issue_board_add_list_v13_6.png differ diff --git a/doc/user/project/img/issue_board_assignee_lists.png b/doc/user/project/img/issue_board_assignee_lists.png deleted file mode 100644 index f2660cd8f80..00000000000 Binary files a/doc/user/project/img/issue_board_assignee_lists.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_assignee_lists_v13_6.png b/doc/user/project/img/issue_board_assignee_lists_v13_6.png new file mode 100644 index 00000000000..d0fbb0a2ef0 Binary files /dev/null and b/doc/user/project/img/issue_board_assignee_lists_v13_6.png differ diff --git a/doc/user/project/img/issue_board_creation.png b/doc/user/project/img/issue_board_creation.png deleted file mode 100644 index 099fe6eee21..00000000000 Binary files a/doc/user/project/img/issue_board_creation.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_creation_v13_6.png b/doc/user/project/img/issue_board_creation_v13_6.png new file mode 100644 index 00000000000..e36b53418fd Binary files /dev/null and b/doc/user/project/img/issue_board_creation_v13_6.png differ diff --git a/doc/user/project/img/issue_board_edit_button.png b/doc/user/project/img/issue_board_edit_button.png deleted file mode 100644 index a0dc6f41592..00000000000 Binary files a/doc/user/project/img/issue_board_edit_button.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_focus_mode.gif b/doc/user/project/img/issue_board_focus_mode.gif deleted file mode 100644 index 9565bdb0865..00000000000 Binary files a/doc/user/project/img/issue_board_focus_mode.gif and /dev/null differ diff --git a/doc/user/project/img/issue_board_milestone_lists.png b/doc/user/project/img/issue_board_milestone_lists.png deleted file mode 100644 index 91926f58f87..00000000000 Binary files a/doc/user/project/img/issue_board_milestone_lists.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_milestone_lists_v13_6.png b/doc/user/project/img/issue_board_milestone_lists_v13_6.png new file mode 100644 index 00000000000..a7718ffd66c Binary files /dev/null and b/doc/user/project/img/issue_board_milestone_lists_v13_6.png differ diff --git a/doc/user/project/img/issue_board_move_issue_card_list.png b/doc/user/project/img/issue_board_move_issue_card_list.png deleted file mode 100644 index 13750a63766..00000000000 Binary files a/doc/user/project/img/issue_board_move_issue_card_list.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_move_issue_card_list_v13_6.png b/doc/user/project/img/issue_board_move_issue_card_list_v13_6.png new file mode 100644 index 00000000000..2b661a63d7d Binary files /dev/null and b/doc/user/project/img/issue_board_move_issue_card_list_v13_6.png differ diff --git a/doc/user/project/img/issue_board_summed_weights.png b/doc/user/project/img/issue_board_summed_weights.png deleted file mode 100644 index 6035d7ca330..00000000000 Binary files a/doc/user/project/img/issue_board_summed_weights.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_summed_weights_v13_6.png b/doc/user/project/img/issue_board_summed_weights_v13_6.png new file mode 100644 index 00000000000..a6482e73c0a Binary files /dev/null and b/doc/user/project/img/issue_board_summed_weights_v13_6.png differ diff --git a/doc/user/project/img/issue_board_system_notes.png b/doc/user/project/img/issue_board_system_notes.png deleted file mode 100644 index c6ecb498198..00000000000 Binary files a/doc/user/project/img/issue_board_system_notes.png and /dev/null differ diff --git a/doc/user/project/img/issue_board_system_notes_v13_6.png b/doc/user/project/img/issue_board_system_notes_v13_6.png new file mode 100644 index 00000000000..4958f63541e Binary files /dev/null and b/doc/user/project/img/issue_board_system_notes_v13_6.png differ diff --git a/doc/user/project/img/issue_board_view_scope.png b/doc/user/project/img/issue_board_view_scope.png deleted file mode 100644 index d173679a0e7..00000000000 Binary files a/doc/user/project/img/issue_board_view_scope.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_add_issues_modal.png b/doc/user/project/img/issue_boards_add_issues_modal.png deleted file mode 100644 index ecddf6709d0..00000000000 Binary files a/doc/user/project/img/issue_boards_add_issues_modal.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_add_issues_modal_v13_6.png b/doc/user/project/img/issue_boards_add_issues_modal_v13_6.png new file mode 100644 index 00000000000..a138efc9c1c Binary files /dev/null and b/doc/user/project/img/issue_boards_add_issues_modal_v13_6.png differ diff --git a/doc/user/project/img/issue_boards_blocked_icon_v12_8.png b/doc/user/project/img/issue_boards_blocked_icon_v12_8.png deleted file mode 100644 index 1055fb48322..00000000000 Binary files a/doc/user/project/img/issue_boards_blocked_icon_v12_8.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_blocked_icon_v13_6.png b/doc/user/project/img/issue_boards_blocked_icon_v13_6.png new file mode 100644 index 00000000000..2dac6bb2ee3 Binary files /dev/null and b/doc/user/project/img/issue_boards_blocked_icon_v13_6.png differ diff --git a/doc/user/project/img/issue_boards_core.png b/doc/user/project/img/issue_boards_core.png deleted file mode 100644 index 41ddbb24b14..00000000000 Binary files a/doc/user/project/img/issue_boards_core.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_core_v13_6.png b/doc/user/project/img/issue_boards_core_v13_6.png new file mode 100644 index 00000000000..8695b523c12 Binary files /dev/null and b/doc/user/project/img/issue_boards_core_v13_6.png differ diff --git a/doc/user/project/img/issue_boards_multiple.png b/doc/user/project/img/issue_boards_multiple.png deleted file mode 100644 index e6183360610..00000000000 Binary files a/doc/user/project/img/issue_boards_multiple.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_multiple_v13_6.png b/doc/user/project/img/issue_boards_multiple_v13_6.png new file mode 100644 index 00000000000..18ff5e2bc66 Binary files /dev/null and b/doc/user/project/img/issue_boards_multiple_v13_6.png differ diff --git a/doc/user/project/img/issue_boards_premium.png b/doc/user/project/img/issue_boards_premium.png deleted file mode 100644 index ef9f5bbea32..00000000000 Binary files a/doc/user/project/img/issue_boards_premium.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_premium_v13_6.png b/doc/user/project/img/issue_boards_premium_v13_6.png new file mode 100644 index 00000000000..8d1c1299d5c Binary files /dev/null and b/doc/user/project/img/issue_boards_premium_v13_6.png differ diff --git a/doc/user/project/img/issue_boards_remove_issue.png b/doc/user/project/img/issue_boards_remove_issue.png deleted file mode 100644 index 7050e6c3ede..00000000000 Binary files a/doc/user/project/img/issue_boards_remove_issue.png and /dev/null differ diff --git a/doc/user/project/img/issue_boards_remove_issue_v13_6.png b/doc/user/project/img/issue_boards_remove_issue_v13_6.png new file mode 100644 index 00000000000..c980759ad0c Binary files /dev/null and b/doc/user/project/img/issue_boards_remove_issue_v13_6.png differ diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index bce40e9a838..4ab8f87e5d4 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -31,7 +31,7 @@ To let your team members organize their own workflows, use [multiple issue boards](#use-cases-for-multiple-issue-boards). This allows creating multiple issue boards in the same project. -![GitLab issue board - Core](img/issue_boards_core.png) +![GitLab issue board - Core](img/issue_boards_core_v13_6.png) Different issue board features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), as shown in the following table: @@ -45,7 +45,7 @@ as shown in the following table: To learn more, visit [GitLab Enterprise features for issue boards](#gitlab-enterprise-features-for-issue-boards) below. -![GitLab issue board - Premium](img/issue_boards_premium.png) +![GitLab issue board - Premium](img/issue_boards_premium_v13_6.png) Watch a [video presentation](https://youtu.be/vjccjHI7aGI) of @@ -69,8 +69,8 @@ For example, let's consider this simplified development workflow: 1. When frontend is complete, the new feature is deployed to a **staging** environment to be tested. 1. When successful, it's deployed to **production**. -If you have the labels "**backend**", "**frontend**", "**staging**", and -"**production**", and an issue board with a list for each, you can: +If you have the labels **Backend**, **Frontend**, **Staging**, and +**Production**, and an issue board with a list for each, you can: - Visualize the entire flow of implementations since the beginning of the development life cycle until deployed to production. @@ -78,7 +78,7 @@ If you have the labels "**backend**", "**frontend**", "**staging**", and - Move issues between lists to organize them according to the labels you've set. - Add multiple issues to lists in the board by selecting one or more existing issues. -![issue card moving](img/issue_board_move_issue_card_list.png) +![issue card moving](img/issue_board_move_issue_card_list_v13_6.png) ### Use cases for multiple issue boards @@ -199,7 +199,7 @@ Using the search box at the top of the menu, you can filter the listed boards. When you have ten or more boards available, a **Recent** section is also shown in the menu, with shortcuts to your last four visited boards. -![Multiple issue boards](img/issue_boards_multiple.png) +![Multiple issue boards](img/issue_boards_multiple_v13_6.png) When you're revisiting an issue board in a project or group with multiple boards, GitLab automatically loads the last board you visited. @@ -229,20 +229,16 @@ An issue board can be associated with a GitLab [Milestone](milestones/index.md#m which automatically filter the board issues accordingly. This allows you to create unique boards according to your team's need. -![Create scoped board](img/issue_board_creation.png) +![Create scoped board](img/issue_board_creation_v13_6.png) -You can define the scope of your board when creating it or by clicking the "Edit board" button. -Once a milestone, assignee or weight is assigned to an issue board, you can no longer +You can define the scope of your board when creating it or by clicking the **Edit board** button. +After a milestone, assignee or weight is assigned to an issue board, you can no longer filter through these in the search bar. In order to do that, you need to remove the desired scope (for example, milestone, assignee, or weight) from the issue board. -![Edit board configuration](img/issue_board_edit_button.png) - If you don't have editing permission in a board, you're still able to see the configuration by clicking **View scope**. -![Viewing board configuration](img/issue_board_view_scope.png) - Watch a [video presentation](https://youtu.be/m5UTNCSqaDk) of the Configurable Issue Board feature. @@ -253,12 +249,8 @@ the Configurable Issue Board feature. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28597) to the Free tier of GitLab.com in 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212331) to GitLab Core in 13.0. -Click the button at the top right to toggle focus mode on and off. In focus mode, the navigation UI -is hidden, allowing you to focus on issues in the board. - -![Board focus mode](img/issue_board_focus_mode.gif) - ---- +To enable or disable focus mode, select the **Toggle focus mode** button (**{maximize}**) at the top +right. In focus mode, the navigation UI is hidden, allowing you to focus on issues in the board. ### Sum of issue weights **(STARTER)** @@ -266,7 +258,7 @@ The top of each list indicates the sum of issue weights for the issues that belong to that list. This is useful when using boards for capacity allocation, especially in combination with [assignee lists](#assignee-lists). -![issue board summed weights](img/issue_board_summed_weights.png) +![issue board summed weights](img/issue_board_summed_weights_v13_6.png) ### Group issue boards **(PREMIUM)** @@ -279,8 +271,6 @@ group and its descendant subgroups. Similarly, you can only filter by group labe boards. When updating milestones and labels for an issue through the sidebar update mechanism, again only group-level objects are available. -![Group issue board](img/group_issue_board.png) - ### Assignee lists **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. @@ -290,15 +280,15 @@ an assignee list that shows all issues assigned to a user. You can have a board with both label lists and assignee lists. To add an assignee list: -1. Click **Add list**. +1. Select the **Add list** dropdown button. 1. Select the **Assignee list** tab. -1. Search and click the user you want to add as an assignee. +1. Search and select the user you want to add as an assignee. Now that the assignee list is added, you can assign or unassign issues to that user by [dragging issues](#drag-issues-between-lists) to and from an assignee list. To remove an assignee list, just as with a label list, click the trash icon. -![Assignee lists](img/issue_board_assignee_lists.png) +![Assignee lists](img/issue_board_assignee_lists_v13_6.png) ### Milestone lists **(PREMIUM)** @@ -307,7 +297,7 @@ To remove an assignee list, just as with a label list, click the trash icon. You're also able to create lists of a milestone. These are lists that filter issues by the assigned milestone, giving you more freedom and visibility on the issue board. To add a milestone list: -1. Click **Add list**. +1. Select the **Add list** dropdown button. 1. Select the **Milestone** tab. 1. Search and click the milestone. @@ -315,7 +305,7 @@ Like the assignee lists, you're able to [drag issues](#drag-issues-between-lists to and from a milestone list to manipulate the milestone of the dragged issues. As in other list types, click the trash icon to remove a list. -![Milestone lists](img/issue_board_milestone_lists.png) +![Milestone lists](img/issue_board_milestone_lists_v13_6.png) ## Work In Progress limits **(STARTER)** @@ -347,7 +337,7 @@ To set a WIP limit for a list: If an issue is blocked by another issue, an icon appears next to its title to indicate its blocked status. -![Blocked issues](img/issue_boards_blocked_icon_v12_8.png) +![Blocked issues](img/issue_boards_blocked_icon_v13_6.png) ## Actions you can take on an issue board @@ -381,16 +371,16 @@ have that label. ### Create a new list -Create a new list by clicking the **Add list** button in the upper right corner of the issue board. +Create a new list by clicking the **Add list** dropdown button in the upper right corner of the issue board. -![creating a new list in an issue board](img/issue_board_add_list.png) +![creating a new list in an issue board](img/issue_board_add_list_v13_6.png) -Then, choose the label or user to create the list from. The new list is inserted -at the end of the lists, before **Done**. Moving and reordering lists is as -easy as dragging them around. +Then, choose the label or user to base the new list on. The new list is inserted +at the end of the lists, before **Done**. To move and reorder lists, drag them around. To create a list for a label that doesn't yet exist, create the label by -choosing **Create new label**. This creates the label immediately and adds it to the dropdown. +choosing **Create project label** or **Create group label**. +This creates the label immediately and adds it to the dropdown. You can now choose it to create a list. ### Delete a list @@ -404,14 +394,14 @@ list view that's removed. You can always restore it later if you need. ### Add issues to a list You can add issues to a list by clicking the **Add issues** button -present in the upper right corner of the issue board. This opens up a modal +in the top right corner of the issue board. This opens up a modal window where you can see all the issues that do not belong to any list. Select one or more issues by clicking the cards and then click **Add issues** to add them to the selected list. You can limit the issues you want to add to the list by filtering by author, assignee, milestone, and label. -![Bulk adding issues to lists](img/issue_boards_add_issues_modal.png) +![Bulk adding issues to lists](img/issue_boards_add_issues_modal_v13_6.png) ### Remove an issue from a list @@ -419,13 +409,13 @@ Removing an issue from a list can be done by clicking the issue card and then clicking the **Remove from board** button in the sidebar. The respective label is removed. -![Remove issue from list](img/issue_boards_remove_issue.png) +![Remove issue from list](img/issue_boards_remove_issue_v13_6.png) ### Filter issues You should be able to use the filters on top of your issue board to show only -the results you want. It's similar to the filtering used in the issue tracker -since the metadata from the issues and labels are re-used in the issue board. +the results you want. It's similar to the filtering used in the issue tracker, +as the metadata from the issues and labels is re-used in the issue board. You can filter by author, assignee, milestone, and label. @@ -435,13 +425,13 @@ By reordering your lists, you can create workflows. As lists in issue boards are based on labels, it works out of the box with your existing issues. So if you've already labeled things with **Backend** and **Frontend**, the issue appears in -the lists as you create them. In addition, this means you can easily move -something between lists by changing a label. +the lists as you create them. In addition, this means you can move something between lists by +changing a label. A typical workflow of using an issue board would be: 1. You have [created](labels.md#label-management) and [prioritized](labels.md#label-priority) - labels so that you can easily categorize your issues. + labels to categorize your issues. 1. You have a bunch of issues (ideally labeled). 1. You visit the issue board and start [creating lists](#create-a-new-list) to create a workflow. @@ -457,15 +447,15 @@ For example, you can create a list based on the label of **Frontend** and one fo **Frontend** list. That way, everyone knows that this issue is now being worked on by the designers. -Then, once they're done, all they have to do is +Then, when they're done, all they have to do is drag it to the next list, **Backend**. Then, a backend developer can -eventually pick it up. Once they’re done, they move it to **Done**, to close the +eventually pick it up. When they’re done, they move it to **Done**, to close the issue. This process can be seen clearly when visiting an issue. With every move to another list, the label changes and a system note is recorded. -![issue board system notes](img/issue_board_system_notes.png) +![issue board system notes](img/issue_board_system_notes_v13_6.png) ### Drag issues between lists @@ -473,16 +463,17 @@ When dragging issues between lists, different behavior occurs depending on the s | | To Open | To Closed | To label `B` list | To assignee `Bob` list | |----------------------------|--------------------|--------------|------------------------------|---------------------------------------| -| From Open | - | Issue closed | `B` added | `Bob` assigned | -| From Closed | Issue reopened | - | Issue reopened
`B` added | Issue reopened
`Bob` assigned | -| From label `A` list | `A` removed | Issue closed | `A` removed
`B` added | `Bob` assigned | -| From assignee `Alice` list | `Alice` unassigned | Issue closed | `B` added | `Alice` unassigned
`Bob` assigned | +| **From Open** | - | Issue closed | `B` added | `Bob` assigned | +| **From Closed** | Issue reopened | - | Issue reopened
`B` added | Issue reopened
`Bob` assigned | +| **From label `A` list** | `A` removed | Issue closed | `A` removed
`B` added | `Bob` assigned | +| **From assignee `Alice` list** | `Alice` unassigned | Issue closed | `B` added | `Alice` unassigned
`Bob` assigned | ### Multi-select issue cards > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18954) in GitLab 12.4. -You can select multiple issue cards, then drag the group to another position within the list, or to another list. This makes it faster to reorder many issues at once. +You can select multiple issue cards, then drag the group to another position within the list, or to +another list. This makes it faster to reorder many issues at once. To select and move multiple cards: diff --git a/doc/user/project/repository/img/repository_mirroring_push_settings.png b/doc/user/project/repository/img/repository_mirroring_push_settings.png index 9fc25dd3b25..8916d21d515 100644 Binary files a/doc/user/project/repository/img/repository_mirroring_push_settings.png and b/doc/user/project/repository/img/repository_mirroring_push_settings.png differ diff --git a/doc/user/project/wiki/img/wiki_sidebar_v13_5.png b/doc/user/project/wiki/img/wiki_sidebar_v13_5.png index 0f445d61d71..f22424749a5 100644 Binary files a/doc/user/project/wiki/img/wiki_sidebar_v13_5.png and b/doc/user/project/wiki/img/wiki_sidebar_v13_5.png differ diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index ed5ecc17a8b..c4040878e02 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -64,8 +64,8 @@ The Advanced Search Syntax also supports the use of filters. The available filte - extension: Filters by extension in the filename. Please write the extension without a leading dot. Exact match only. - blob: Filters by Git `object ID`. Exact match only. -To use them, simply add them to your query in the format `:` without - any spaces between the colon (`:`) and the value. +To use them, add them to your keyword in the format `:` without +any spaces between the colon (`:`) and the value. A keyword or an asterisk (`*`) is required for filter searches and has to be added in front of the filter separated by a space. Examples: diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb new file mode 100644 index 00000000000..bc0a181a06c --- /dev/null +++ b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates missing dismissal information for + # vulnerability entries. + class PopulateMissingVulnerabilityDismissalInformation + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + + has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding' + + scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } + + def copy_dismissal_information + return unless finding&.dismissal_feedback + + update_columns( + dismissed_at: finding.dismissal_feedback.created_at, + dismissed_by_id: finding.dismissal_feedback.author_id + ) + end + end + + class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation + include ShaAttribute + + self.table_name = 'vulnerability_occurrences' + + sha_attribute :project_fingerprint + + def dismissal_feedback + Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first + end + end + + class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation + DISMISSAL_TYPE = 0 + + self.table_name = 'vulnerability_feedback' + + scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } + end + + def perform(*vulnerability_ids) + Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) } + + log_info(vulnerability_ids) + end + + private + + def populate_for(vulnerability) + log_warning(vulnerability) unless vulnerability.copy_dismissal_information + rescue StandardError => error + log_error(error, vulnerability) + end + + def log_info(vulnerability_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Dismissal information has been copied', + count: vulnerability_ids.length + ) + end + + def log_warning(vulnerability) + ::Gitlab::BackgroundMigration::Logger.warn( + migrator: self.class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability.id + ) + end + + def log_error(error, vulnerability) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + vulnerability_id: vulnerability.id + ) + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 97ae6c4ceba..7198163216e 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -33,6 +33,7 @@ module Gitlab locations .compact .map(&method(:normalize_location)) + .flat_map(&method(:expand_project_files)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -52,6 +53,15 @@ module Gitlab end end + def expand_project_files(location) + return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: false) + return location unless location[:project] + + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb new file mode 100644 index 00000000000..cedc2db2a1a --- /dev/null +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'faker' + +module QA + RSpec.describe 'Verify', :runner, :requires_admin, :skip_live_env do + describe "Include multiple files from a project" do + let(:feature_flag) { :ci_include_multiple_files_from_project } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:expected_text) { Faker::Lorem.sentence } + let(:unexpected_text) { Faker::Lorem.sentence } + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-pipeline-1' + end + end + + let(:other_project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-pipeline-2' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.project = project + runner.name = executor + runner.tags = [executor] + end + end + + before do + Runtime::Feature.enable(feature_flag) + Flow::Login.sign_in + add_included_files + add_main_ci_file + project.visit! + view_the_last_pipeline + end + + after do + Runtime::Feature.disable(feature_flag) + runner.remove_via_api! + end + + it 'runs the pipeline with composed config', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1082' do + Page::Project::Pipeline::Show.perform do |pipeline| + aggregate_failures 'pipeline has all expected jobs' do + expect(pipeline).to have_job('build') + expect(pipeline).to have_job('test') + expect(pipeline).to have_job('deploy') + end + + pipeline.click_job('test') + end + + Page::Project::Job::Show.perform do |job| + aggregate_failures 'main CI is not overridden' do + expect(job.output).to have_no_content("#{unexpected_text}") + expect(job.output).to have_content("#{expected_text}") + end + end + end + + private + + def add_main_ci_file + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add config file' + commit.add_files([main_ci_file]) + end + end + + def add_included_files + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = other_project + commit.commit_message = 'Add files' + commit.add_files([included_file_1, included_file_2]) + end + end + + def view_the_last_pipeline + Page::Project::Menu.perform(&:click_ci_cd_pipelines) + Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success) + Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline) + end + + def main_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + include: + - project: #{other_project.full_path} + file: + - file1.yml + - file2.yml + + build: + stage: build + tags: ["#{executor}"] + script: echo 'build' + + test: + stage: test + tags: ["#{executor}"] + script: echo "#{expected_text}" + YAML + } + end + + def included_file_1 + { + file_path: 'file1.yml', + content: <<~YAML + test: + stage: test + tags: ["#{executor}"] + script: echo "#{unexpected_text}" + YAML + } + end + + def included_file_2 + { + file_path: 'file2.yml', + content: <<~YAML + deploy: + stage: deploy + tags: ["#{executor}"] + script: echo 'deploy' + YAML + } + end + end + end +end diff --git a/spec/graphql/types/issue_type_spec.rb b/spec/graphql/types/issue_type_spec.rb index b8f1c4a0fcc..c483eb66f12 100644 --- a/spec/graphql/types/issue_type_spec.rb +++ b/spec/graphql/types/issue_type_spec.rb @@ -15,7 +15,7 @@ RSpec.describe GitlabSchema.types['Issue'] do it 'has specific fields' do fields = %i[id iid title description state reference author assignees updated_by participants labels milestone due_date - confidential discussion_locked upvotes downvotes user_notes_count web_path web_url relative_position + confidential discussion_locked upvotes downvotes user_notes_count user_discussions_count web_path web_url relative_position subscribed time_estimate total_time_spent human_time_estimate human_total_time_spent closed_at created_at updated_at task_completion_status designs design_collection alert_management_alert severity current_user_todos] diff --git a/spec/graphql/types/merge_request_type_spec.rb b/spec/graphql/types/merge_request_type_spec.rb index 9d901655b7b..a7512506100 100644 --- a/spec/graphql/types/merge_request_type_spec.rb +++ b/spec/graphql/types/merge_request_type_spec.rb @@ -17,7 +17,7 @@ RSpec.describe GitlabSchema.types['MergeRequest'] do description_html state created_at updated_at source_project target_project project project_id source_project_id target_project_id source_branch target_branch work_in_progress merge_when_pipeline_succeeds diff_head_sha - merge_commit_sha user_notes_count should_remove_source_branch + merge_commit_sha user_notes_count user_discussions_count should_remove_source_branch diff_refs diff_stats diff_stats_summary force_remove_source_branch merge_status in_progress_merge_commit_sha merge_error allow_collaboration should_be_rebased rebase_commit_sha diff --git a/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb b/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb new file mode 100644 index 00000000000..44c5f3d1381 --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation, schema: 20201028160832 do + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:findings) { table(:vulnerability_occurrences) } + let(:scanners) { table(:vulnerability_scanners) } + let(:identifiers) { table(:vulnerability_identifiers) } + let(:feedback) { table(:vulnerability_feedback) } + + let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } + let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } + let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') } + let(:vulnerability_1) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } + let(:vulnerability_2) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') } + let(:identifier) { identifiers.create!(project_id: project.id, fingerprint: 'foo', external_type: 'bar', external_id: 'zoo', name: 'identifier') } + + before do + feedback.create!(feedback_type: 0, + category: 'sast', + project_fingerprint: '418291a26024a1445b23fe64de9380cdcdfd1fa8', + project_id: project.id, + author_id: user.id, + created_at: Time.current) + + findings.create!(name: 'Finding', + report_type: 'sast', + project_fingerprint: Gitlab::Database::ShaAttribute.new.serialize('418291a26024a1445b23fe64de9380cdcdfd1fa8'), + location_fingerprint: 'bar', + severity: 1, + confidence: 1, + metadata_version: 1, + raw_metadata: '', + uuid: SecureRandom.uuid, + project_id: project.id, + vulnerability_id: vulnerability_1.id, + scanner_id: scanner.id, + primary_identifier_id: identifier.id) + + allow(::Gitlab::BackgroundMigration::Logger).to receive_messages(info: true, warn: true, error: true) + end + + describe '#perform' do + it 'updates the missing dismissal information of the vulnerability' do + expect { subject.perform(vulnerability_1.id, vulnerability_2.id) }.to change { vulnerability_1.reload.dismissed_at }.from(nil) + .and change { vulnerability_1.reload.dismissed_by_id }.from(nil).to(user.id) + end + + it 'writes log messages' do + subject.perform(vulnerability_1.id, vulnerability_2.id) + + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:info).with(migrator: described_class.name, + message: 'Dismissal information has been copied', + count: 2) + expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(migrator: described_class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability_2.id) + end + end +end diff --git a/spec/lib/gitlab/ci/config/external/mapper_spec.rb b/spec/lib/gitlab/ci/config/external/mapper_spec.rb index bf14d8d6b34..7ad57827e30 100644 --- a/spec/lib/gitlab/ci/config/external/mapper_spec.rb +++ b/spec/lib/gitlab/ci/config/external/mapper_spec.rb @@ -100,6 +100,42 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do expect { subject }.to raise_error(described_class::AmbigiousSpecificationError) end end + + context "when the key is a project's file" do + let(:values) do + { include: { project: project.full_path, file: local_file }, + image: 'ruby:2.7' } + end + + it 'returns File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Project)) + end + end + + context "when the key is project's files" do + let(:values) do + { include: { project: project.full_path, file: [local_file, 'another_file_path.yml'] }, + image: 'ruby:2.7' } + end + + it 'returns two File instances' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Project), + an_instance_of(Gitlab::Ci::Config::External::File::Project)) + end + + context 'when FF ci_include_multiple_files_from_project is disabled' do + before do + stub_feature_flags(ci_include_multiple_files_from_project: false) + end + + it 'returns a File instance' do + expect(subject).to contain_exactly( + an_instance_of(Gitlab::Ci::Config::External::File::Project)) + end + end + end end context "when 'include' is defined as an array" do @@ -161,6 +197,16 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'raises an exception' do expect { subject }.to raise_error(described_class::DuplicateIncludesError) end + + context 'when including multiple files from a project' do + let(:values) do + { include: { project: project.full_path, file: [local_file, local_file] } } + end + + it 'raises an exception' do + expect { subject }.to raise_error(described_class::DuplicateIncludesError) + end + end end context "when too many 'includes' are defined" do @@ -179,6 +225,16 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do it 'raises an exception' do expect { subject }.to raise_error(described_class::TooManyIncludesError) end + + context 'when including multiple files from a project' do + let(:values) do + { include: { project: project.full_path, file: [local_file, 'another_file_path.yml'] } } + end + + it 'raises an exception' do + expect { subject }.to raise_error(described_class::TooManyIncludesError) + end + end end end end diff --git a/spec/lib/gitlab/ci/config/external/processor_spec.rb b/spec/lib/gitlab/ci/config/external/processor_spec.rb index 9786e050399..150a2ec2929 100644 --- a/spec/lib/gitlab/ci/config/external/processor_spec.rb +++ b/spec/lib/gitlab/ci/config/external/processor_spec.rb @@ -302,5 +302,82 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do end end end + + context 'when a valid project file is defined' do + let(:values) do + { + include: { project: another_project.full_path, file: '/templates/my-build.yml' }, + image: 'ruby:2.7' + } + end + + before do + another_project.add_developer(user) + + allow_next_instance_of(Repository) do |repository| + allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do + <<~HEREDOC + my_build: + script: echo Hello World + HEREDOC + end + end + end + + it 'appends the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :my_build]) + end + end + + context 'when valid project files are defined in a single include' do + let(:values) do + { + include: { + project: another_project.full_path, + file: ['/templates/my-build.yml', '/templates/my-test.yml'] + }, + image: 'ruby:2.7' + } + end + + before do + another_project.add_developer(user) + + allow_next_instance_of(Repository) do |repository| + allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do + <<~HEREDOC + my_build: + script: echo Hello World + HEREDOC + end + + allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-test.yml') do + <<~HEREDOC + my_test: + script: echo Hello World + HEREDOC + end + end + end + + it 'appends the file to the values' do + output = processor.perform + expect(output.keys).to match_array([:image, :my_build, :my_test]) + end + + context 'when FF ci_include_multiple_files_from_project is disabled' do + before do + stub_feature_flags(ci_include_multiple_files_from_project: false) + end + + it 'raises an error' do + expect { processor.perform }.to raise_error( + described_class::IncludeError, + 'Included file `["/templates/my-build.yml", "/templates/my-test.yml"]` needs to be a string' + ) + end + end + end end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 41a45fe4ab7..b5a0f0e3fd7 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -246,6 +246,14 @@ RSpec.describe Gitlab::Ci::Config do let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' } let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' } + let(:local_file_content) do + File.read(Rails.root.join(local_location)) + end + + let(:local_location_hash) do + YAML.safe_load(local_file_content).deep_symbolize_keys + end + let(:remote_file_content) do <<~HEREDOC variables: @@ -256,8 +264,8 @@ RSpec.describe Gitlab::Ci::Config do HEREDOC end - let(:local_file_content) do - File.read(Rails.root.join(local_location)) + let(:remote_file_hash) do + YAML.safe_load(remote_file_content).deep_symbolize_keys end let(:gitlab_ci_yml) do @@ -283,22 +291,11 @@ RSpec.describe Gitlab::Ci::Config do context "when gitlab_ci_yml has valid 'include' defined" do it 'returns a composed hash' do - before_script_values = [ - "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v", - "which ruby", - "bundle install --jobs $(nproc) \"${FLAGS[@]}\"" - ] - variables = { - POSTGRES_USER: "user", - POSTGRES_PASSWORD: "testing-password", - POSTGRES_ENABLED: "true", - POSTGRES_DB: "$CI_ENVIRONMENT_SLUG" - } composed_hash = { - before_script: before_script_values, + before_script: local_location_hash[:before_script], image: "ruby:2.7", rspec: { script: ["bundle exec rspec"] }, - variables: variables + variables: remote_file_hash[:variables] } expect(config.to_hash).to eq(composed_hash) @@ -575,5 +572,56 @@ RSpec.describe Gitlab::Ci::Config do ) end end + + context "when including multiple files from a project" do + let(:other_file_location) { 'my_builds.yml' } + + let(:other_file_content) do + <<~HEREDOC + build: + stage: build + script: echo hello + + rspec: + stage: test + script: bundle exec rspec + HEREDOC + end + + let(:gitlab_ci_yml) do + <<~HEREDOC + include: + - project: #{project.full_path} + file: + - #{local_location} + - #{other_file_location} + + image: ruby:2.7 + HEREDOC + end + + before do + project.add_developer(user) + + allow_next_instance_of(Repository) do |repository| + allow(repository).to receive(:blob_data_at).with(an_instance_of(String), local_location) + .and_return(local_file_content) + + allow(repository).to receive(:blob_data_at).with(an_instance_of(String), other_file_location) + .and_return(other_file_content) + end + end + + it 'returns a composed hash' do + composed_hash = { + before_script: local_location_hash[:before_script], + image: "ruby:2.7", + build: { stage: "build", script: "echo hello" }, + rspec: { stage: "test", script: "bundle exec rspec" } + } + + expect(config.to_hash).to eq(composed_hash) + end + end end end diff --git a/spec/migrations/schedule_populate_missing_dismissal_information_for_vulnerabilities_spec.rb b/spec/migrations/schedule_populate_missing_dismissal_information_for_vulnerabilities_spec.rb new file mode 100644 index 00000000000..e5934f2171f --- /dev/null +++ b/spec/migrations/schedule_populate_missing_dismissal_information_for_vulnerabilities_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe SchedulePopulateMissingDismissalInformationForVulnerabilities do + let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:user) { users.create!(name: 'test', email: 'test@example.com', projects_limit: 5) } + let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') } + let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') } + + let!(:vulnerability_1) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } + let!(:vulnerability_2) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id, dismissed_at: Time.now) } + let!(:vulnerability_3) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id, dismissed_by_id: user.id) } + let!(:vulnerability_4) { vulnerabilities.create!(title: 'title', state: 2, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id, dismissed_at: Time.now, dismissed_by_id: user.id) } + let!(:vulnerability_5) { vulnerabilities.create!(title: 'title', state: 1, severity: 0, confidence: 5, report_type: 2, project_id: project.id, author_id: user.id) } + + around do |example| + freeze_time { Sidekiq::Testing.fake! { example.run } } + end + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 1) + end + + it 'schedules the background jobs', :aggregate_failures do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to be(3) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(3.minutes, vulnerability_1.id) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(6.minutes, vulnerability_2.id) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(9.minutes, vulnerability_3.id) + end +end diff --git a/spec/requests/api/graphql/project/issues_spec.rb b/spec/requests/api/graphql/project/issues_spec.rb index 40fec6ba068..4f27f08bf98 100644 --- a/spec/requests/api/graphql/project/issues_spec.rb +++ b/spec/requests/api/graphql/project/issues_spec.rb @@ -9,10 +9,9 @@ RSpec.describe 'getting an issue list for a project' do let_it_be(:project) { create(:project, :repository, :public) } let_it_be(:current_user) { create(:user) } - let_it_be(:issues, reload: true) do - [create(:issue, project: project, discussion_locked: true), - create(:issue, :with_alert, project: project)] - end + let_it_be(:issue_a, reload: true) { create(:issue, project: project, discussion_locked: true) } + let_it_be(:issue_b, reload: true) { create(:issue, :with_alert, project: project) } + let_it_be(:issues, reload: true) { [issue_a, issue_b] } let(:fields) do <<~QUERY @@ -414,4 +413,42 @@ RSpec.describe 'getting an issue list for a project' do expect(response_assignee_ids(issues_data)).to match_array(assignees_as_global_ids(new_issues)) end end + + describe 'N+1 query checks' do + let(:extra_iid_for_second_query) { issue_b.iid.to_s } + let(:search_params) { { iids: [issue_a.iid.to_s] } } + + def execute_query + query = graphql_query_for( + :project, + { full_path: project.full_path }, + query_graphql_field(:issues, search_params, [ + query_graphql_field(:nodes, nil, requested_fields) + ]) + ) + post_graphql(query, current_user: current_user) + end + + context 'when requesting `user_notes_count`' do + let(:requested_fields) { [:user_notes_count] } + + before do + create_list(:note_on_issue, 2, noteable: issue_a, project: project) + create(:note_on_issue, noteable: issue_b, project: project) + end + + include_examples 'N+1 query check' + end + + context 'when requesting `user_discussions_count`' do + let(:requested_fields) { [:user_discussions_count] } + + before do + create_list(:note_on_issue, 2, noteable: issue_a, project: project) + create(:note_on_issue, noteable: issue_b, project: project) + end + + include_examples 'N+1 query check' + end + end end diff --git a/spec/requests/api/graphql/project/merge_requests_spec.rb b/spec/requests/api/graphql/project/merge_requests_spec.rb index c737e0b8caf..2b8d537f9fc 100644 --- a/spec/requests/api/graphql/project/merge_requests_spec.rb +++ b/spec/requests/api/graphql/project/merge_requests_spec.rb @@ -243,6 +243,17 @@ RSpec.describe 'getting merge request listings nested in a project' do include_examples 'N+1 query check' end + + context 'when requesting `user_discussions_count`' do + let(:requested_fields) { [:user_discussions_count] } + + before do + create_list(:note_on_merge_request, 2, noteable: merge_request_a, project: project) + create(:note_on_merge_request, noteable: merge_request_c, project: project) + end + + include_examples 'N+1 query check' + end end describe 'sorting and pagination' do diff --git a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb index fb6cdf55be3..8df9b0c3e60 100644 --- a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb +++ b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb @@ -279,6 +279,40 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do end end end + + context 'when specifying multiple files' do + let(:config) do + <<~YAML + test: + script: rspec + deploy: + variables: + CROSS: downstream + stage: deploy + trigger: + include: + - project: my-namespace/my-project + file: + - 'path/to/child1.yml' + - 'path/to/child2.yml' + YAML + end + + it_behaves_like 'successful creation' do + let(:expected_bridge_options) do + { + 'trigger' => { + 'include' => [ + { + 'file' => ["path/to/child1.yml", "path/to/child2.yml"], + 'project' => 'my-namespace/my-project' + } + ] + } + } + end + end + end end end