Add latest changes from gitlab-org/gitlab@master

.gitpod.yml

@@ -66,6 +66,8 @@ tasks:
         sleep 5
         printf "$(date) – GitLab is up (took ~%.1f minutes)\n" "$((10*$SECONDS/60))e-1" | tee -a /workspace/startup.log
         gp preview $(gp url 3000) || true
+        # Speed up backend tests
+        export GITLAB_TEST_EAGER_LOAD=false
       )
 
 ports:

app/assets/javascripts/diffs/components/diff_stats.vue

@@ -62,8 +62,8 @@ export default {
     </div>
     <div v-else class="diff-stats-contents">
       <div v-if="hasDiffFiles" class="diff-stats-group">
-        <gl-icon name="doc-code" class="diff-stats-icon text-secondary" />
-        <span class="text-secondary bold">{{ diffFilesCountText }} {{ filesText }}</span>
+        <gl-icon name="doc-code" class="diff-stats-icon gl-text-gray-500" />
+        <span class="gl-text-gray-500 bold">{{ diffFilesCountText }} {{ filesText }}</span>
       </div>
       <div
         class="diff-stats-group gl-text-green-600 gl-display-flex gl-align-items-center"

app/models/alert_management/alert.rb

@@ -9,6 +9,7 @@ module AlertManagement
     include ShaAttribute
     include Sortable
     include Noteable
+    include Mentionable
     include Gitlab::SQL::Pattern
     include Presentable
     include Gitlab::Utils::StrongMemoize

app/models/commit.rb

@@ -3,6 +3,7 @@
 class Commit
   extend ActiveModel::Naming
   extend Gitlab::Cache::RequestCache
+  extend Gitlab::Utils::Override
 
   include ActiveModel::Conversion
   include Noteable
@@ -327,7 +328,7 @@ class Commit
   end
 
   def user_mentions
-    CommitUserMention.where(commit_id: self.id)
+    user_mention_class.where(commit_id: self.id)
   end
 
   def discussion_notes
@@ -554,6 +555,19 @@ class Commit
     Ability.allowed?(user, :read_commit, self)
   end
 
+  override :user_mention_class
+  def user_mention_class
+    CommitUserMention
+  end
+
+  override :user_mention_identifier
+  def user_mention_identifier
+    {
+      commit_id: id,
+      note_id: nil
+    }
+  end
+
   private
 
   def expire_note_etag_cache_for_related_mrs

app/models/concerns/cache_markdown_field.rb

@@ -160,6 +160,8 @@ module CacheMarkdownField
     # We can only store mentions if the mentionable is a database object
     return unless self.is_a?(ApplicationRecord)
 
+    return store_mentions_without_subtransaction! if Feature.enabled?(:store_mentions_without_subtransaction, default_enabled: :yaml)
+
     refs = all_references(self.author)
 
     references = {}
@@ -190,6 +192,29 @@ module CacheMarkdownField
     true
   end
 
+  def store_mentions_without_subtransaction!
+    identifier = user_mention_identifier
+
+    # this may happen due to notes polymorphism, so noteable_id may point to a record
+    # that no longer exists as we cannot have FK on noteable_id
+    return if identifier.blank?
+
+    refs = all_references(self.author)
+
+    references = {}
+    references[:mentioned_users_ids] = refs.mentioned_user_ids.presence
+    references[:mentioned_groups_ids] = refs.mentioned_group_ids.presence
+    references[:mentioned_projects_ids] = refs.mentioned_project_ids.presence
+
+    if references.compact.any?
+      user_mention_class.upsert(references.merge(identifier), unique_by: identifier.compact.keys)
+    else
+      user_mention_class.delete_by(identifier)
+    end
+
+    true
+  end
+
   def mentionable_attributes_changed?(changes = saved_changes)
     return false unless is_a?(Mentionable)
 

app/models/concerns/enums/ci/pipeline.rb

@@ -38,7 +38,8 @@ module Enums
           external_pull_request_event: 11,
           parent_pipeline: 12,
           ondemand_dast_scan: 13,
-          ondemand_dast_validation: 14
+          ondemand_dast_validation: 14,
+          security_orchestration_policy: 15
         }
       end
 
@@ -52,7 +53,7 @@ module Enums
       # - when an ondemand_dast_validation pipeline runs it is for validating a DAST site
       #   profile and should not affect the ref CI status.
       def self.dangling_sources
-        sources.slice(:webide, :parent_pipeline, :ondemand_dast_scan, :ondemand_dast_validation)
+        sources.slice(:webide, :parent_pipeline, :ondemand_dast_scan, :ondemand_dast_validation, :security_orchestration_policy)
       end
 
       # CI sources are those pipeline events that affect the CI status of the ref

app/models/concerns/mentionable.rb

@@ -161,6 +161,21 @@ module Mentionable
     create_cross_references!(author)
   end
 
+  def user_mention_class
+    user_mention_association.klass
+  end
+
+  # Identifier for the user mention that is parsed from model description rather then its related notes.
+  # Models that have a description attribute like Issue, MergeRequest, Epic, Snippet may have such a user mention.
+  # Other mentionable models like DesignManagement::Design, will never have such record as those do not have
+  # a description attribute.
+  def user_mention_identifier
+    {
+      user_mention_association.foreign_key => id,
+      note_id: nil
+    }
+  end
+
   private
 
   def extracted_mentionables(refs)
@@ -199,6 +214,10 @@ module Mentionable
     {}
   end
 
+  def user_mention_association
+    association(:user_mentions).reflection
+  end
+
   # User mention that is parsed from model description rather then its related notes.
   # Models that have a description attribute like Issue, MergeRequest, Epic, Snippet may have such a user mention.
   # Other mentionable models like Commit, DesignManagement::Design, will never have such record as those do not have

app/models/note.rb

@@ -5,6 +5,8 @@
 # A note of this type is never resolvable.
 class Note < ApplicationRecord
   extend ActiveModel::Naming
+  extend Gitlab::Utils::Override
+
   include Gitlab::Utils::StrongMemoize
   include Participable
   include Mentionable
@@ -583,6 +585,22 @@ class Note < ApplicationRecord
     cache_key_items.join(':')
   end
 
+  override :user_mention_class
+  def user_mention_class
+    return if noteable.blank?
+
+    noteable.user_mention_class
+  end
+
+  override :user_mention_identifier
+  def user_mention_identifier
+    return if noteable.blank?
+
+    noteable.user_mention_identifier.merge({
+      note_id: id
+    })
+  end
+
   private
 
   # Using this method followed by a call to *save* may result in *ActiveRecord::RecordNotUnique* exception

config/feature_flags/development/store_mentions_without_subtransaction.yml

@@ -0,0 +1,8 @@
+---
+name: store_mentions_without_subtransaction
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68433
+rollout_issue_url: 
+milestone: '14.3'
+type: development
+group: group::project management
+default_enabled: false

db/migrate/20210818055357_add_unique_commit_design_user_mention_indexes.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class AddUniqueCommitDesignUserMentionIndexes < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  COMMIT_INDEX_NAME = 'commit_id_and_note_id_index'
+  DESIGN_INDEX_NAME = 'design_user_mentions_on_design_id_and_note_id_index'
+
+  COMMIT_UNIQUE_INDEX_NAME = 'commit_user_mentions_on_commit_id_and_note_id_unique_index'
+  DESIGN_UNIQUE_INDEX_NAME = 'design_user_mentions_on_design_id_and_note_id_unique_index'
+
+  def up
+    add_concurrent_index :commit_user_mentions, [:commit_id, :note_id], unique: true, name: COMMIT_UNIQUE_INDEX_NAME
+    add_concurrent_index :design_user_mentions, [:design_id, :note_id], unique: true, name: DESIGN_UNIQUE_INDEX_NAME
+
+    remove_concurrent_index_by_name :commit_user_mentions, COMMIT_INDEX_NAME
+    remove_concurrent_index_by_name :design_user_mentions, DESIGN_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :design_user_mentions, [:design_id, :note_id], name: DESIGN_INDEX_NAME
+    add_concurrent_index :commit_user_mentions, [:commit_id, :note_id], name: COMMIT_INDEX_NAME
+
+    remove_concurrent_index_by_name :design_user_mentions, DESIGN_UNIQUE_INDEX_NAME
+    remove_concurrent_index_by_name :commit_user_mentions, COMMIT_UNIQUE_INDEX_NAME
+  end
+end

db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb

@@ -3,31 +3,11 @@
 class ScheduleCopyCiBuildsColumnsToSecurityScans < ActiveRecord::Migration[6.1]
   include Gitlab::Database::MigrationHelpers
 
-  INTERVAL = 2.minutes.to_i
-  BATCH_SIZE = 5_000
-  MIGRATION = 'CopyCiBuildsColumnsToSecurityScans'
-
-  disable_ddl_transaction!
-
-  class SecurityScan < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'security_scans'
-  end
-
   def up
-    SecurityScan.reset_column_information
-
-    queue_background_migration_jobs_by_range_at_intervals(
-      SecurityScan,
-      MIGRATION,
-      INTERVAL,
-      batch_size: BATCH_SIZE,
-      track_jobs: true
-    )
+    # no-op as we found an issue with bg migration, we fixed it and rescheduling it again.
   end
 
   def down
-    # noop
+    # no-op
   end
 end

db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class ScheduleCopyCiBuildsColumnsToSecurityScans2 < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 5_000
+  MIGRATION = 'CopyCiBuildsColumnsToSecurityScans'
+
+  disable_ddl_transaction!
+
+  class SecurityScan < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'security_scans'
+  end
+
+  def up
+    SecurityScan.reset_column_information
+
+    delete_job_tracking(MIGRATION, status: %w[pending succeeded])
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      SecurityScan,
+      MIGRATION,
+      INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # noop
+  end
+end

db/schema_migrations/20210816183304

@@ -0,0 +1 @@
+08fed4e3269629304c5036361c237f4c9b860c26212b155869773296a79df01a

db/schema_migrations/20210818055357

@@ -0,0 +1 @@
+1092a16d742b08ef2ef5f74bdaa92bb5f9cedbdb1161ab71abe501c39b164689

db/structure.sql

@@ -22895,7 +22895,7 @@ CREATE INDEX ci_builds_gitlab_monitor_metrics ON ci_builds USING btree (status,
 
 CREATE INDEX code_owner_approval_required ON protected_branches USING btree (project_id, code_owner_approval_required) WHERE (code_owner_approval_required = true);
 
-CREATE INDEX commit_id_and_note_id_index ON commit_user_mentions USING btree (commit_id, note_id);
+CREATE UNIQUE INDEX commit_user_mentions_on_commit_id_and_note_id_unique_index ON commit_user_mentions USING btree (commit_id, note_id);
 
 CREATE INDEX composer_cache_files_index_on_deleted_at ON packages_composer_cache_files USING btree (delete_at, id);
 
@@ -22905,7 +22905,7 @@ CREATE UNIQUE INDEX dast_site_profiles_builds_on_ci_build_id ON dast_site_profil
 
 CREATE UNIQUE INDEX design_management_designs_versions_uniqueness ON design_management_designs_versions USING btree (design_id, version_id);
 
-CREATE INDEX design_user_mentions_on_design_id_and_note_id_index ON design_user_mentions USING btree (design_id, note_id);
+CREATE UNIQUE INDEX design_user_mentions_on_design_id_and_note_id_unique_index ON design_user_mentions USING btree (design_id, note_id);
 
 CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_and_note_id_index ON epic_user_mentions USING btree (epic_id, note_id);
 

doc/api/vulnerability_exports.md

@@ -198,7 +198,7 @@ The response is `404 Not Found` if the vulnerability export is not finished yet
 Example response:
 
 ```csv
-Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers
+Group Name,Project Name,Tool,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers
 Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997
 Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269
 Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001

doc/ci/pipelines/index.md

@@ -374,8 +374,8 @@ dependencies.
 
 Jobs in the leftmost column run first, and jobs that depend on them are grouped in the next columns.
 
-For example, `build-job2` depends only on jobs in the first column, so it displays
-in the second column from the left. `deploy-job2` depends on jobs in both the first
+For example, `test-job1` depends only on jobs in the first column, so it displays
+in the second column from the left. `deploy-job1` depends on jobs in both the first
 and second column and displays in the third column:
 
 ![jobs grouped by needs dependency](img/pipelines_graph_dependency_view_v13_12.png)

doc/user/application_security/policies/index.md

@@ -129,9 +129,9 @@ rule in the defined policy are met.
 
 | Field | Type | Possible values | Description |
 |-------|------|-----------------|-------------|
-| `scan` | `string` | `dast` | The action's type. |
-| `site_profile` | `string` | Name of the selected [DAST site profile](../dast/index.md#site-profile). | The DAST site profile to execute the DAST scan. |
-| `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/index.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. |
+| `scan` | `string` | `dast`, `secret_detection` | The action's type. |
+| `site_profile` | `string` | Name of the selected [DAST site profile](../dast/index.md#site-profile). | The DAST site profile to execute the DAST scan. This field should only be set if `scan` type is `dast`. |
+| `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/index.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. This field should only be set if `scan` type is `dast`.|
 
 Note the following:
 
@@ -144,6 +144,11 @@ Note the following:
 - When configuring policies with a scheduled DAST scan, the author of the commit in the security
   policy project's repository must have access to the scanner and site profiles. Otherwise, the scan
   is not scheduled successfully.
+- For a secret detection scan, only rules with the default ruleset are supported. [Custom rulesets](../secret_detection/index.md#custom-rulesets)
+  are not supported.
+- A secret detection scan runs in `normal` mode when executed as part of a pipeline, and in
+  [`historic`](../secret_detection/index.md#full-history-secret-scan)
+  mode when executed as part of a scheduled scan.
 
 Here's an example:
 
@@ -161,8 +166,8 @@ scan_execution_policy:
   - scan: dast
     scanner_profile: Scanner Profile A
     site_profile: Site Profile B
-- name: Enforce DAST scan every 10 minutes
-  description: This policy enforces a DAST scan to run every 10 minutes
+- name: Enforce DAST and secret detection scans every 10 minutes
+  description: This policy enforces DAST and secret detection scans to run every 10 minutes
   enabled: true
   rules:
   - type: schedule
@@ -173,12 +178,25 @@ scan_execution_policy:
   - scan: dast
     scanner_profile: Scanner Profile C
     site_profile: Site Profile D
+  - scan: secret_detection
+- name: Enforce Secret Detection in every default branch pipeline
+  description: This policy enforces pipeline configuration to have a job with Secret Detection scan for the default branch
+  enabled: true
+  rules:
+  - type: pipeline
+    branches:
+    - main
+  actions:
+  - scan: secret_detection
 ```
 
-In this example, the DAST scan runs with the scanner profile `Scanner Profile A` and the site
-profile `Site Profile B` for every pipeline executed on branches that match the
-`release/*` wildcard (for example, branch name `release/v1.2.1`); and the DAST scan runs with
-the scanner profile `Scanner Profile C` and the site profile `Site Profile D` every 10 minutes.
+In this example:
+
+- For every pipeline executed on branches that match the `release/*` wildcard (for example, branch
+  `release/v1.2.1`), DAST scans run with `Scanner Profile A` and `Site Profile B`.
+- DAST and secret detection scans run every 10 minutes. The DAST scan runs with `Scanner Profile C`
+  and `Site Profile D`.
+- Secret detection scans run for every pipeline executed on the `main` branch.
 
 ## Security Policy project selection
 

lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb

@@ -30,15 +30,15 @@ module Gitlab
       rescue StandardError => error
         Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
       end
-    end
 
-    private
+      private
 
-    def mark_job_as_succeeded(*arguments)
-      Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
-        'CopyCiBuildsColumnsToSecurityScans',
-        arguments
-      )
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'CopyCiBuildsColumnsToSecurityScans',
+          arguments
+        )
+      end
     end
   end
 end

spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb

@@ -3,12 +3,13 @@
 require 'spec_helper'
 require_migration!
 
-RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
+RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans2 do
   let_it_be(:namespaces) { table(:namespaces) }
   let_it_be(:projects) { table(:projects) }
   let_it_be(:ci_pipelines) { table(:ci_pipelines) }
   let_it_be(:ci_builds) { table(:ci_builds) }
   let_it_be(:security_scans) { table(:security_scans) }
+  let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
 
   let!(:namespace) { namespaces.create!(name: 'namespace', path: 'namespace') }
   let!(:project) { projects.create!(namespace_id: namespace.id) }
@@ -22,6 +23,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
   let!(:scan2) { security_scans.create!(build_id: build2.id, scan_type: 1) }
   let!(:scan3) { security_scans.create!(build_id: build3.id, scan_type: 1) }
 
+  let!(:job_class_name) { described_class::MIGRATION }
+  let!(:tracked_pending_job) { background_migration_jobs.create!(class_name: job_class_name, status: 0, arguments: [1]) }
+  let!(:tracked_successful_job) { background_migration_jobs.create!(class_name: job_class_name, status: 1, arguments: [2]) }
+  let(:jobs) { Gitlab::Database::BackgroundMigrationJob.where(id: [tracked_pending_job.id, tracked_successful_job.id] ).for_migration_class(job_class_name) }
+
   before do
     stub_const("#{described_class}::BATCH_SIZE", 2)
     allow_next_instance_of(Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans) do |instance|
@@ -34,8 +40,11 @@ RSpec.describe ScheduleCopyCiBuildsColumnsToSecurityScans do
   end
 
   it 'schedules background migrations', :aggregate_failures do
+    expect(jobs).not_to be_empty
+
     migrate!
 
+    expect(jobs).to be_empty
     expect(BackgroundMigrationWorker.jobs.size).to eq(2)
     expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, scan1.id, scan2.id)
     expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, scan3.id, scan3.id)

spec/services/issues/update_service_spec.rb

@@ -517,7 +517,7 @@ RSpec.describe Issues::UpdateService, :mailer do
           update_issue(description: "- [x] Task 1 #{user.to_reference}\n- [ ] Task 2 #{user.to_reference}")
         end
 
-        expect(recorded.count).to eq(baseline.count - 1)
+        expect(recorded.count).to eq(baseline.count)
         expect(recorded.cached_count).to eq(0)
       end
     end

spec/support/shared_examples/models/mentionable_shared_examples.rb

@@ -207,7 +207,7 @@ RSpec.shared_examples 'an editable mentionable' do
 end
 
 RSpec.shared_examples 'mentions in description' do |mentionable_type|
-  describe 'when storing user mentions' do
+  shared_examples 'when storing user mentions' do
     before do
       mentionable.store_mentions!
     end
@@ -238,10 +238,26 @@ RSpec.shared_examples 'mentions in description' do |mentionable_type|
       end
     end
   end
+
+  context 'when store_mentions_without_subtransaction is enabled' do
+    before do
+      stub_feature_flags(store_mentions_without_subtransaction: true)
+    end
+
+    it_behaves_like 'when storing user mentions'
+  end
+
+  context 'when store_mentions_without_subtransaction is disabled' do
+    before do
+      stub_feature_flags(store_mentions_without_subtransaction: false)
+    end
+
+    it_behaves_like 'when storing user mentions'
+  end
 end
 
 RSpec.shared_examples 'mentions in notes' do |mentionable_type|
-  context 'when mentionable notes contain mentions' do
+  shared_examples 'when mentionable notes contain mentions' do
     let(:user) { create(:user) }
     let(:user2) { create(:user) }
     let(:group) { create(:group) }
@@ -261,6 +277,22 @@ RSpec.shared_examples 'mentions in notes' do |mentionable_type|
       expect(mentionable.referenced_groups(user)).to eq [group]
     end
   end
+
+  context 'when store_mentions_without_subtransaction is enabled' do
+    before do
+      stub_feature_flags(store_mentions_without_subtransaction: true)
+    end
+
+    it_behaves_like 'when mentionable notes contain mentions'
+  end
+
+  context 'when store_mentions_without_subtransaction is disabled' do
+    before do
+      stub_feature_flags(store_mentions_without_subtransaction: false)
+    end
+
+    it_behaves_like 'when mentionable notes contain mentions'
+  end
 end
 
 RSpec.shared_examples 'load mentions from DB' do |mentionable_type|
@@ -278,7 +310,7 @@ RSpec.shared_examples 'load mentions from DB' do |mentionable_type|
 
     context 'when stored user mention contains ids of inexistent records' do
       before do
-        user_mention = note.send(:model_user_mention)
+        user_mention = note.user_mentions.first
         mention_ids = {
           mentioned_users_ids: user_mention.mentioned_users_ids.to_a << non_existing_record_id,
           mentioned_projects_ids: user_mention.mentioned_projects_ids.to_a << non_existing_record_id,
@@ -302,7 +334,7 @@ RSpec.shared_examples 'load mentions from DB' do |mentionable_type|
       let(:group_member) { create(:group_member, user: create(:user), group: private_group) }
 
       before do
-        user_mention = note.send(:model_user_mention)
+        user_mention = note.user_mentions.first
         mention_ids = {
           mentioned_projects_ids: user_mention.mentioned_projects_ids.to_a << private_project.id,
           mentioned_groups_ids: user_mention.mentioned_groups_ids.to_a << private_group.id