From fca4c56c94bbba41a1cf8a963761b69173b4fd63 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 19 Aug 2021 03:10:47 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitpod.yml | 2 + .../diffs/components/diff_stats.vue | 4 +- app/models/alert_management/alert.rb | 1 + app/models/commit.rb | 16 +++++++- app/models/concerns/cache_markdown_field.rb | 25 ++++++++++++ app/models/concerns/enums/ci/pipeline.rb | 5 ++- app/models/concerns/mentionable.rb | 19 +++++++++ app/models/note.rb | 18 +++++++++ .../store_mentions_without_subtransaction.yml | 8 ++++ ...ique_commit_design_user_mention_indexes.rb | 29 ++++++++++++++ ...opy_ci_builds_columns_to_security_scans.rb | 24 +---------- ...py_ci_builds_columns_to_security_scans2.rb | 35 ++++++++++++++++ db/schema_migrations/20210816183304 | 1 + db/schema_migrations/20210818055357 | 1 + db/structure.sql | 4 +- doc/api/vulnerability_exports.md | 2 +- doc/ci/pipelines/index.md | 4 +- .../application_security/policies/index.md | 36 ++++++++++++----- ...opy_ci_builds_columns_to_security_scans.rb | 14 +++---- ...builds_columns_to_security_scans2_spec.rb} | 11 ++++- spec/services/issues/update_service_spec.rb | 2 +- .../models/mentionable_shared_examples.rb | 40 +++++++++++++++++-- 22 files changed, 247 insertions(+), 54 deletions(-) create mode 100644 config/feature_flags/development/store_mentions_without_subtransaction.yml create mode 100644 db/migrate/20210818055357_add_unique_commit_design_user_mention_indexes.rb create mode 100644 db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb create mode 100644 db/schema_migrations/20210816183304 create mode 100644 db/schema_migrations/20210818055357 rename spec/migrations/{schedule_copy_ci_builds_columns_to_security_scans_spec.rb => schedule_copy_ci_builds_columns_to_security_scans2_spec.rb} (72%) diff --git a/.gitpod.yml b/.gitpod.yml index b47ede87c0a..6b77ee18e1e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -66,6 +66,8 @@ tasks: sleep 5 printf "$(date) – GitLab is up (took ~%.1f minutes)\n" "$((10*$SECONDS/60))e-1" | tee -a /workspace/startup.log gp preview $(gp url 3000) || true + # Speed up backend tests + export GITLAB_TEST_EAGER_LOAD=false ) ports: diff --git a/app/assets/javascripts/diffs/components/diff_stats.vue b/app/assets/javascripts/diffs/components/diff_stats.vue index 05d4fbe7c20..e8b4ff16aec 100644 --- a/app/assets/javascripts/diffs/components/diff_stats.vue +++ b/app/assets/javascripts/diffs/components/diff_stats.vue @@ -62,8 +62,8 @@ export default {
- - {{ diffFilesCountText }} {{ filesText }} + + {{ diffFilesCountText }} {{ filesText }}
id, + note_id: nil + } + end + private def extracted_mentionables(refs) @@ -199,6 +214,10 @@ module Mentionable {} end + def user_mention_association + association(:user_mentions).reflection + end + # User mention that is parsed from model description rather then its related notes. # Models that have a description attribute like Issue, MergeRequest, Epic, Snippet may have such a user mention. # Other mentionable models like Commit, DesignManagement::Design, will never have such record as those do not have diff --git a/app/models/note.rb b/app/models/note.rb index 5d630aa4984..34ffd7c91af 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -5,6 +5,8 @@ # A note of this type is never resolvable. class Note < ApplicationRecord extend ActiveModel::Naming + extend Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize include Participable include Mentionable @@ -583,6 +585,22 @@ class Note < ApplicationRecord cache_key_items.join(':') end + override :user_mention_class + def user_mention_class + return if noteable.blank? + + noteable.user_mention_class + end + + override :user_mention_identifier + def user_mention_identifier + return if noteable.blank? + + noteable.user_mention_identifier.merge({ + note_id: id + }) + end + private # Using this method followed by a call to *save* may result in *ActiveRecord::RecordNotUnique* exception diff --git a/config/feature_flags/development/store_mentions_without_subtransaction.yml b/config/feature_flags/development/store_mentions_without_subtransaction.yml new file mode 100644 index 00000000000..a71aa204935 --- /dev/null +++ b/config/feature_flags/development/store_mentions_without_subtransaction.yml @@ -0,0 +1,8 @@ +--- +name: store_mentions_without_subtransaction +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68433 +rollout_issue_url: +milestone: '14.3' +type: development +group: group::project management +default_enabled: false diff --git a/db/migrate/20210818055357_add_unique_commit_design_user_mention_indexes.rb b/db/migrate/20210818055357_add_unique_commit_design_user_mention_indexes.rb new file mode 100644 index 00000000000..d9d05f2b737 --- /dev/null +++ b/db/migrate/20210818055357_add_unique_commit_design_user_mention_indexes.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AddUniqueCommitDesignUserMentionIndexes < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + COMMIT_INDEX_NAME = 'commit_id_and_note_id_index' + DESIGN_INDEX_NAME = 'design_user_mentions_on_design_id_and_note_id_index' + + COMMIT_UNIQUE_INDEX_NAME = 'commit_user_mentions_on_commit_id_and_note_id_unique_index' + DESIGN_UNIQUE_INDEX_NAME = 'design_user_mentions_on_design_id_and_note_id_unique_index' + + def up + add_concurrent_index :commit_user_mentions, [:commit_id, :note_id], unique: true, name: COMMIT_UNIQUE_INDEX_NAME + add_concurrent_index :design_user_mentions, [:design_id, :note_id], unique: true, name: DESIGN_UNIQUE_INDEX_NAME + + remove_concurrent_index_by_name :commit_user_mentions, COMMIT_INDEX_NAME + remove_concurrent_index_by_name :design_user_mentions, DESIGN_INDEX_NAME + end + + def down + add_concurrent_index :design_user_mentions, [:design_id, :note_id], name: DESIGN_INDEX_NAME + add_concurrent_index :commit_user_mentions, [:commit_id, :note_id], name: COMMIT_INDEX_NAME + + remove_concurrent_index_by_name :design_user_mentions, DESIGN_UNIQUE_INDEX_NAME + remove_concurrent_index_by_name :commit_user_mentions, COMMIT_UNIQUE_INDEX_NAME + end +end diff --git a/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb b/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb index dc90c202c41..106d1430dca 100644 --- a/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb +++ b/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb @@ -3,31 +3,11 @@ class ScheduleCopyCiBuildsColumnsToSecurityScans < ActiveRecord::Migration[6.1] include Gitlab::Database::MigrationHelpers - INTERVAL = 2.minutes.to_i - BATCH_SIZE = 5_000 - MIGRATION = 'CopyCiBuildsColumnsToSecurityScans' - - disable_ddl_transaction! - - class SecurityScan < ActiveRecord::Base - include EachBatch - - self.table_name = 'security_scans' - end - def up - SecurityScan.reset_column_information - - queue_background_migration_jobs_by_range_at_intervals( - SecurityScan, - MIGRATION, - INTERVAL, - batch_size: BATCH_SIZE, - track_jobs: true - ) + # no-op as we found an issue with bg migration, we fixed it and rescheduling it again. end def down - # noop + # no-op end end diff --git a/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb b/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb new file mode 100644 index 00000000000..b8f843fdd50 --- /dev/null +++ b/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class ScheduleCopyCiBuildsColumnsToSecurityScans2 < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + INTERVAL = 2.minutes.to_i + BATCH_SIZE = 5_000 + MIGRATION = 'CopyCiBuildsColumnsToSecurityScans' + + disable_ddl_transaction! + + class SecurityScan < ActiveRecord::Base + include EachBatch + + self.table_name = 'security_scans' + end + + def up + SecurityScan.reset_column_information + + delete_job_tracking(MIGRATION, status: %w[pending succeeded]) + + queue_background_migration_jobs_by_range_at_intervals( + SecurityScan, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # noop + end +end diff --git a/db/schema_migrations/20210816183304 b/db/schema_migrations/20210816183304 new file mode 100644 index 00000000000..eebc52f99d7 --- /dev/null +++ b/db/schema_migrations/20210816183304 @@ -0,0 +1 @@ +08fed4e3269629304c5036361c237f4c9b860c26212b155869773296a79df01a \ No newline at end of file diff --git a/db/schema_migrations/20210818055357 b/db/schema_migrations/20210818055357 new file mode 100644 index 00000000000..1557aa0ac3e --- /dev/null +++ b/db/schema_migrations/20210818055357 @@ -0,0 +1 @@ +1092a16d742b08ef2ef5f74bdaa92bb5f9cedbdb1161ab71abe501c39b164689 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index d6474efffc5..eee73636eb1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -22895,7 +22895,7 @@ CREATE INDEX ci_builds_gitlab_monitor_metrics ON ci_builds USING btree (status, CREATE INDEX code_owner_approval_required ON protected_branches USING btree (project_id, code_owner_approval_required) WHERE (code_owner_approval_required = true); -CREATE INDEX commit_id_and_note_id_index ON commit_user_mentions USING btree (commit_id, note_id); +CREATE UNIQUE INDEX commit_user_mentions_on_commit_id_and_note_id_unique_index ON commit_user_mentions USING btree (commit_id, note_id); CREATE INDEX composer_cache_files_index_on_deleted_at ON packages_composer_cache_files USING btree (delete_at, id); @@ -22905,7 +22905,7 @@ CREATE UNIQUE INDEX dast_site_profiles_builds_on_ci_build_id ON dast_site_profil CREATE UNIQUE INDEX design_management_designs_versions_uniqueness ON design_management_designs_versions USING btree (design_id, version_id); -CREATE INDEX design_user_mentions_on_design_id_and_note_id_index ON design_user_mentions USING btree (design_id, note_id); +CREATE UNIQUE INDEX design_user_mentions_on_design_id_and_note_id_unique_index ON design_user_mentions USING btree (design_id, note_id); CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_and_note_id_index ON epic_user_mentions USING btree (epic_id, note_id); diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index 4240d363ff0..efe0ea000a7 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -198,7 +198,7 @@ The response is `404 Not Found` if the vulnerability export is not finished yet Example response: ```csv -Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers +Group Name,Project Name,Tool,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 22f18235f99..1ac3854abdf 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -374,8 +374,8 @@ dependencies. Jobs in the leftmost column run first, and jobs that depend on them are grouped in the next columns. -For example, `build-job2` depends only on jobs in the first column, so it displays -in the second column from the left. `deploy-job2` depends on jobs in both the first +For example, `test-job1` depends only on jobs in the first column, so it displays +in the second column from the left. `deploy-job1` depends on jobs in both the first and second column and displays in the third column: ![jobs grouped by needs dependency](img/pipelines_graph_dependency_view_v13_12.png) diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index 076872c9864..3d0135678b7 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -129,9 +129,9 @@ rule in the defined policy are met. | Field | Type | Possible values | Description | |-------|------|-----------------|-------------| -| `scan` | `string` | `dast` | The action's type. | -| `site_profile` | `string` | Name of the selected [DAST site profile](../dast/index.md#site-profile). | The DAST site profile to execute the DAST scan. | -| `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/index.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. | +| `scan` | `string` | `dast`, `secret_detection` | The action's type. | +| `site_profile` | `string` | Name of the selected [DAST site profile](../dast/index.md#site-profile). | The DAST site profile to execute the DAST scan. This field should only be set if `scan` type is `dast`. | +| `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/index.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. This field should only be set if `scan` type is `dast`.| Note the following: @@ -144,6 +144,11 @@ Note the following: - When configuring policies with a scheduled DAST scan, the author of the commit in the security policy project's repository must have access to the scanner and site profiles. Otherwise, the scan is not scheduled successfully. +- For a secret detection scan, only rules with the default ruleset are supported. [Custom rulesets](../secret_detection/index.md#custom-rulesets) + are not supported. +- A secret detection scan runs in `normal` mode when executed as part of a pipeline, and in + [`historic`](../secret_detection/index.md#full-history-secret-scan) + mode when executed as part of a scheduled scan. Here's an example: @@ -161,8 +166,8 @@ scan_execution_policy: - scan: dast scanner_profile: Scanner Profile A site_profile: Site Profile B -- name: Enforce DAST scan every 10 minutes - description: This policy enforces a DAST scan to run every 10 minutes +- name: Enforce DAST and secret detection scans every 10 minutes + description: This policy enforces DAST and secret detection scans to run every 10 minutes enabled: true rules: - type: schedule @@ -173,12 +178,25 @@ scan_execution_policy: - scan: dast scanner_profile: Scanner Profile C site_profile: Site Profile D + - scan: secret_detection +- name: Enforce Secret Detection in every default branch pipeline + description: This policy enforces pipeline configuration to have a job with Secret Detection scan for the default branch + enabled: true + rules: + - type: pipeline + branches: + - main + actions: + - scan: secret_detection ``` -In this example, the DAST scan runs with the scanner profile `Scanner Profile A` and the site -profile `Site Profile B` for every pipeline executed on branches that match the -`release/*` wildcard (for example, branch name `release/v1.2.1`); and the DAST scan runs with -the scanner profile `Scanner Profile C` and the site profile `Site Profile D` every 10 minutes. +In this example: + +- For every pipeline executed on branches that match the `release/*` wildcard (for example, branch + `release/v1.2.1`), DAST scans run with `Scanner Profile A` and `Site Profile B`. +- DAST and secret detection scans run every 10 minutes. The DAST scan runs with `Scanner Profile C` + and `Site Profile D`. +- Secret detection scans run for every pipeline executed on the `main` branch. ## Security Policy project selection diff --git a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb index 59ce06d3e8f..107ac9b0c3b 100644 --- a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb +++ b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb @@ -30,15 +30,15 @@ module Gitlab rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end - end - private + private - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'CopyCiBuildsColumnsToSecurityScans', - arguments - ) + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'CopyCiBuildsColumnsToSecurityScans', + arguments + ) + end end end end diff --git a/spec/migrations/schedule_copy_ci_builds_columns_to_security_scans_spec.rb b/spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb similarity index 72% rename from spec/migrations/schedule_copy_ci_builds_columns_to_security_scans_spec.rb rename to spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb index 5ebd8753892..012c7d065fc 100644 --- a/spec/migrations/schedule_copy_ci_builds_columns_to_security_scans_spec.rb +++ b/spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb @@ -3,12 +3,13 @@ require 'spec_helper' require_migration! -RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do +RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans2 do let_it_be(:namespaces) { table(:namespaces) } let_it_be(:projects) { table(:projects) } let_it_be(:ci_pipelines) { table(:ci_pipelines) } let_it_be(:ci_builds) { table(:ci_builds) } let_it_be(:security_scans) { table(:security_scans) } + let_it_be(:background_migration_jobs) { table(:background_migration_jobs) } let!(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') } let!(:project) { projects.create!(namespace_id: namespace.id) } @@ -22,6 +23,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do let!(:scan2) { security_scans.create!(build_id: build2.id, scan_type: 1) } let!(:scan3) { security_scans.create!(build_id: build3.id, scan_type: 1) } + let!(:job_class_name) { described_class::MIGRATION } + let!(:tracked_pending_job) { background_migration_jobs.create!(class_name: job_class_name, status: 0, arguments: [1]) } + let!(:tracked_successful_job) { background_migration_jobs.create!(class_name: job_class_name, status: 1, arguments: [2]) } + let(:jobs) { Gitlab::Database::BackgroundMigrationJob.where(id: [tracked_pending_job.id, tracked_successful_job.id] ).for_migration_class(job_class_name) } + before do stub_const("#{described_class}::BATCH_SIZE", 2) allow_next_instance_of(Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans) do |instance| @@ -34,8 +40,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do end it 'schedules background migrations', :aggregate_failures do + expect(jobs).not_to be_empty + migrate! + expect(jobs).to be_empty expect(BackgroundMigrationWorker.jobs.size).to eq(2) expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, scan1.id, scan2.id) expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, scan3.id, scan3.id) diff --git a/spec/services/issues/update_service_spec.rb b/spec/services/issues/update_service_spec.rb index 1e922401028..29ac7df88eb 100644 --- a/spec/services/issues/update_service_spec.rb +++ b/spec/services/issues/update_service_spec.rb @@ -517,7 +517,7 @@ RSpec.describe Issues::UpdateService, :mailer do update_issue(description: "- [x] Task 1 #{user.to_reference}\n- [ ] Task 2 #{user.to_reference}") end - expect(recorded.count).to eq(baseline.count - 1) + expect(recorded.count).to eq(baseline.count) expect(recorded.cached_count).to eq(0) end end diff --git a/spec/support/shared_examples/models/mentionable_shared_examples.rb b/spec/support/shared_examples/models/mentionable_shared_examples.rb index 04630484964..07c5f730e95 100644 --- a/spec/support/shared_examples/models/mentionable_shared_examples.rb +++ b/spec/support/shared_examples/models/mentionable_shared_examples.rb @@ -207,7 +207,7 @@ RSpec.shared_examples 'an editable mentionable' do end RSpec.shared_examples 'mentions in description' do |mentionable_type| - describe 'when storing user mentions' do + shared_examples 'when storing user mentions' do before do mentionable.store_mentions! end @@ -238,10 +238,26 @@ RSpec.shared_examples 'mentions in description' do |mentionable_type| end end end + + context 'when store_mentions_without_subtransaction is enabled' do + before do + stub_feature_flags(store_mentions_without_subtransaction: true) + end + + it_behaves_like 'when storing user mentions' + end + + context 'when store_mentions_without_subtransaction is disabled' do + before do + stub_feature_flags(store_mentions_without_subtransaction: false) + end + + it_behaves_like 'when storing user mentions' + end end RSpec.shared_examples 'mentions in notes' do |mentionable_type| - context 'when mentionable notes contain mentions' do + shared_examples 'when mentionable notes contain mentions' do let(:user) { create(:user) } let(:user2) { create(:user) } let(:group) { create(:group) } @@ -261,6 +277,22 @@ RSpec.shared_examples 'mentions in notes' do |mentionable_type| expect(mentionable.referenced_groups(user)).to eq [group] end end + + context 'when store_mentions_without_subtransaction is enabled' do + before do + stub_feature_flags(store_mentions_without_subtransaction: true) + end + + it_behaves_like 'when mentionable notes contain mentions' + end + + context 'when store_mentions_without_subtransaction is disabled' do + before do + stub_feature_flags(store_mentions_without_subtransaction: false) + end + + it_behaves_like 'when mentionable notes contain mentions' + end end RSpec.shared_examples 'load mentions from DB' do |mentionable_type| @@ -278,7 +310,7 @@ RSpec.shared_examples 'load mentions from DB' do |mentionable_type| context 'when stored user mention contains ids of inexistent records' do before do - user_mention = note.send(:model_user_mention) + user_mention = note.user_mentions.first mention_ids = { mentioned_users_ids: user_mention.mentioned_users_ids.to_a << non_existing_record_id, mentioned_projects_ids: user_mention.mentioned_projects_ids.to_a << non_existing_record_id, @@ -302,7 +334,7 @@ RSpec.shared_examples 'load mentions from DB' do |mentionable_type| let(:group_member) { create(:group_member, user: create(:user), group: private_group) } before do - user_mention = note.send(:model_user_mention) + user_mention = note.user_mentions.first mention_ids = { mentioned_projects_ids: user_mention.mentioned_projects_ids.to_a << private_project.id, mentioned_groups_ids: user_mention.mentioned_groups_ids.to_a << private_group.id