Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -133,6 +133,7 @@ update-storybook-yarn-cache:
     paths:
       - tmp/tests/frontend/
       - knapsack/
+      - crystalball/
 
 # Builds FOSS, and EE fixtures in the EE project.
 # Builds FOSS fixtures in the FOSS project.

GITALY_SERVER_VERSION

@@ -1 +1 @@
-06ec7a17f320497d13efdc06f7798b919f45fa9d
+8d7e242576249154697fed7876cd9fe2a31ffdc3

app/models/loose_foreign_keys/deleted_record.rb

@@ -4,29 +4,29 @@ class LooseForeignKeys::DeletedRecord < ApplicationRecord
   self.primary_key = :id
 
   scope :for_table, -> (table) { where(fully_qualified_table_name: table) }
-  scope :ordered_by_id, -> { order(:id, :primary_key_value) }
-  # This needs to be parameterized once we start adding partitions
-  scope :for_partition, -> { where(partition: 1) }
+  scope :consume_order, -> { order(:partition, :consume_after, :id) }
 
   enum status: { pending: 1, processed: 2 }, _prefix: :status
 
   def self.load_batch_for_table(table, batch_size)
     for_table(table)
-      .for_partition
       .status_pending
-      .ordered_by_id
+      .consume_order
       .limit(batch_size)
       .to_a
   end
 
-  def self.mark_records_processed_for_table_between(table, from_record, to_record)
-    from = from_record.id
-    to = to_record.id
+  def self.mark_records_processed(all_records)
+    # Run a query for each partition to optimize the row lookup by primary key (partition, id)
+    update_count = 0
 
-    for_table(table)
-      .for_partition
-      .status_pending
-      .where(id: from..to)
-      .update_all(status: :processed)
+    all_records.group_by(&:partition).each do |partition, records_within_partition|
+      update_count += status_pending
+        .where(partition: partition)
+        .where(id: records_within_partition.pluck(:id))
+        .update_all(status: :processed)
+    end
+
+    update_count
   end
 end

app/services/labels/transfer_service.rb

@@ -50,21 +50,32 @@ module Labels
 
     # rubocop: disable CodeReuse/ActiveRecord
     def group_labels_applied_to_issues
-      @group_labels_applied_to_issues ||= Label.joins(:issues)
-        .where(
-          issues: { project_id: project.id },
-          labels: { group_id: old_group.self_and_ancestors }
-        )
+      @labels_applied_to_issues ||= if use_optimized_group_labels_query?
+                                      Label.joins(:issues)
+                                        .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" )
+                                        .where(issues: { project_id: project.id }).reorder(nil)
+                                    else
+                                      Label.joins(:issues).where(
+                                        issues: { project_id: project.id },
+                                        labels: { group_id: old_group.self_and_ancestors }
+                                      )
+                                    end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
     # rubocop: disable CodeReuse/ActiveRecord
     def group_labels_applied_to_merge_requests
-      @group_labels_applied_to_merge_requests ||= Label.joins(:merge_requests)
-        .where(
-          merge_requests: { target_project_id: project.id },
-          labels: { group_id: old_group.self_and_ancestors }
-        )
+      @labels_applied_to_mrs ||= if use_optimized_group_labels_query?
+                                   Label.joins(:merge_requests)
+                                     .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" )
+                                     .where(merge_requests: { target_project_id: project.id }).reorder(nil)
+                                 else
+                                   Label.joins(:merge_requests)
+                                     .where(
+                                       merge_requests: { target_project_id: project.id },
+                                       labels: { group_id: old_group.self_and_ancestors }
+                                     )
+                                 end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
@@ -88,5 +99,9 @@ module Labels
         .update_all(label_id: new_label_id)
     end
     # rubocop: enable CodeReuse/ActiveRecord
+
+    def use_optimized_group_labels_query?
+      Feature.enabled?(:use_optimized_group_labels_query, project.root_namespace, default_enabled: :yaml)
+    end
   end
 end

app/services/loose_foreign_keys/batch_cleaner_service.rb

@@ -25,8 +25,7 @@ module LooseForeignKeys
       return if modification_tracker.over_limit?
 
       # At this point, all associations are cleaned up, we can update the status of the parent records
-      update_count = LooseForeignKeys::DeletedRecord
-        .mark_records_processed_for_table_between(deleted_parent_records.first.fully_qualified_table_name, deleted_parent_records.first, deleted_parent_records.last)
+      update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records)
 
       deleted_records_counter.increment({ table: parent_klass.table_name, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count)
     end

config/feature_flags/development/use_optimized_group_labels_query.yml

@@ -0,0 +1,8 @@
+---
+name: use_optimized_group_labels_query
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73501
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344957
+milestone: '14.5'
+type: development
+group: group::workspace
+default_enabled: false

data/deprecations/disable_strict_host_key_checking.yml

@@ -3,11 +3,10 @@
   announcement_date: "2021-11-22"
   removal_milestone: "15.0" # the milestone when this feature is planned to be removed
   body: | # Do not modify this line, instead modify the lines below.
-    In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner config.toml. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor.
+    In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor.
 
     In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor.
   stage: Verify
   tiers: [Core, Premium, Ultimate]
   issue_url: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28192
   documentation_url: https://docs.gitlab.com/runner/executors/ssh.html#security
-

db/migrate/20211109100050_add_consume_after_to_loose_fk_deleted_records.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddConsumeAfterToLooseFkDeletedRecords < Gitlab::Database::Migration[1.0]
+  enable_lock_retries!
+
+  def up
+    add_column :loose_foreign_keys_deleted_records, :consume_after, :datetime_with_timezone, default: -> { 'NOW()' }
+  end
+
+  def down
+    remove_column :loose_foreign_keys_deleted_records, :consume_after
+  end
+end

db/migrate/20211109101010_support_partition_query_in_loose_fk_table.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class SupportPartitionQueryInLooseFkTable < Gitlab::Database::Migration[1.0]
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_loose_foreign_keys_deleted_records_for_partitioned_query'
+
+  def up
+    add_concurrent_partitioned_index :loose_foreign_keys_deleted_records,
+      %I[partition fully_qualified_table_name consume_after id],
+      where: 'status = 1',
+      name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_partitioned_index_by_name :loose_foreign_keys_deleted_records, INDEX_NAME
+  end
+end

db/post_migrate/20211029102822_add_open_source_plan.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class AddOpenSourcePlan < Gitlab::Database::Migration[1.0]
+  class Plan < ActiveRecord::Base
+    self.inheritance_column = :_type_disabled
+
+    has_one :limits, class_name: 'PlanLimits'
+
+    def actual_limits
+      self.limits || self.build_limits
+    end
+  end
+
+  class PlanLimits < ActiveRecord::Base
+    self.inheritance_column = :_type_disabled
+
+    belongs_to :plan
+  end
+
+  def create_plan_limits(plan_limit_name, plan)
+    plan_limit = Plan.find_or_initialize_by(name: plan_limit_name).actual_limits.dup
+    plan_limit.plan = plan
+    plan_limit.save!
+  end
+
+  def up
+    return unless Gitlab.dev_env_or_com?
+
+    opensource = Plan.create!(name: 'opensource', title: 'Open Source Program')
+
+    create_plan_limits('ultimate', opensource)
+  end
+
+  def down
+    return unless Gitlab.dev_env_or_com?
+
+    Plan.where(name: 'opensource').delete_all
+  end
+end

db/post_migrate/20211109112454_drop_old_loose_fk_deleted_records_index.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class DropOldLooseFkDeletedRecordsIndex < Gitlab::Database::Migration[1.0]
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_loose_foreign_keys_deleted_records_for_loading_records'
+
+  def up
+    remove_concurrent_partitioned_index_by_name :loose_foreign_keys_deleted_records, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_partitioned_index :loose_foreign_keys_deleted_records,
+      %I[fully_qualified_table_name id primary_key_value partition],
+      where: 'status = 1',
+      name: INDEX_NAME
+  end
+end

db/schema_migrations/20211029102822

@@ -0,0 +1 @@
+e1f9d87287048010e9816fd5b4a9a2d30b64d2ad150226852f6679b950031914

db/schema_migrations/20211109100050

@@ -0,0 +1 @@
+86aa6ad1759a00c2cc5cb6dc2e381aead2910a24f0e37933a5e72af56d08101a

db/schema_migrations/20211109101010

@@ -0,0 +1 @@
+30eb98b8fdb24bc5de357b0ec14a6b92d520db025c82bd7b9448f71542c7d7e3

db/schema_migrations/20211109112454

@@ -0,0 +1 @@
+1bc48cdae55eea5a5963edd3a138d7d6859afa6caafe0b793c553fdfabe9f488

db/structure.sql

@@ -1021,6 +1021,7 @@ CREATE TABLE loose_foreign_keys_deleted_records (
     status smallint DEFAULT 1 NOT NULL,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
     fully_qualified_table_name text NOT NULL,
+    consume_after timestamp with time zone DEFAULT now(),
     CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150))
 )
 PARTITION BY LIST (partition);
@@ -1041,6 +1042,7 @@ CREATE TABLE gitlab_partitions_static.loose_foreign_keys_deleted_records_1 (
     status smallint DEFAULT 1 NOT NULL,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
     fully_qualified_table_name text NOT NULL,
+    consume_after timestamp with time zone DEFAULT now(),
     CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150))
 );
 ALTER TABLE ONLY loose_foreign_keys_deleted_records ATTACH PARTITION gitlab_partitions_static.loose_foreign_keys_deleted_records_1 FOR VALUES IN ('1');
@@ -24073,6 +24075,10 @@ CREATE INDEX index_merge_request_stage_events_project_duration ON ONLY analytics
 
 CREATE INDEX index_006f943df6 ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_16 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
 
+CREATE INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ON ONLY loose_foreign_keys_deleted_records USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1);
+
+CREATE INDEX index_01e3390fac ON gitlab_partitions_static.loose_foreign_keys_deleted_records_1 USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1);
+
 CREATE INDEX index_02749b504c ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_11 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
 
 CREATE INDEX index_merge_request_stage_events_group_duration ON ONLY analytics_cycle_analytics_merge_request_stage_events USING btree (stage_event_hash_id, group_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
@@ -24391,10 +24397,6 @@ CREATE INDEX index_8a0fc3de4f ON gitlab_partitions_static.analytics_cycle_analyt
 
 CREATE INDEX index_8b9f9a19a4 ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_18 USING btree (stage_event_hash_id, group_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
 
-CREATE INDEX index_loose_foreign_keys_deleted_records_for_loading_records ON ONLY loose_foreign_keys_deleted_records USING btree (fully_qualified_table_name, id, primary_key_value, partition) WHERE (status = 1);
-
-CREATE INDEX index_8be8640437 ON gitlab_partitions_static.loose_foreign_keys_deleted_records_1 USING btree (fully_qualified_table_name, id, primary_key_value, partition) WHERE (status = 1);
-
 CREATE INDEX index_8fb48e72ce ON gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_26 USING btree (stage_event_hash_id, group_id, end_event_timestamp, issue_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
 
 CREATE INDEX index_9201b952a0 ON gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_13 USING btree (stage_event_hash_id, group_id, end_event_timestamp, issue_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL);
@@ -27911,6 +27913,8 @@ ALTER INDEX index_issue_stage_events_project_duration ATTACH PARTITION gitlab_pa
 
 ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_006f943df6;
 
+ALTER INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ATTACH PARTITION gitlab_partitions_static.index_01e3390fac;
+
 ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_02749b504c;
 
 ALTER INDEX index_merge_request_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_0287f5ba09;
@@ -28217,8 +28221,6 @@ ALTER INDEX index_issue_stage_events_project_in_progress_duration ATTACH PARTITI
 
 ALTER INDEX index_merge_request_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_8b9f9a19a4;
 
-ALTER INDEX index_loose_foreign_keys_deleted_records_for_loading_records ATTACH PARTITION gitlab_partitions_static.index_8be8640437;
-
 ALTER INDEX index_issue_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_8fb48e72ce;
 
 ALTER INDEX index_issue_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_9201b952a0;

doc/update/deprecations.md

@@ -66,6 +66,14 @@ We decided to remove the GitLab Serverless features as they never really resonat
 
 Announced: 2021-09-22
 
+### Known host required for GitLab Runner SSH executor
+
+In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor.
+
+In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor.
+
+Announced: 2021-11-22
+
 ### Legacy database configuration
 
 The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html)

doc/user/application_security/secret_detection/index.md

@@ -202,8 +202,9 @@ Secret Detection can be customized by defining available CI/CD variables:
 
 | CI/CD variable                    | Default value | Description |
 |-----------------------------------|---------------|-------------|
-| `SECRET_DETECTION_COMMIT_FROM`    | -             | The commit a Gitleaks scan starts at. |
-| `SECRET_DETECTION_COMMIT_TO`      | -             | The commit a Gitleaks scan ends at. |
+| `SECRET_DETECTION_COMMIT_FROM`    | -             | The commit a Gitleaks scan starts at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. |
+| `SECRET_DETECTION_COMMIT_TO`      | -             | The commit a Gitleaks scan ends at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. |
+| `SECRET_DETECTION_COMMITS`        | -             | The list of commits that Gitleaks should scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. |
 | `SECRET_DETECTION_EXCLUDED_PATHS` | ""            | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. |
 | `SECRET_DETECTION_HISTORIC_SCAN`  | false         | Flag to enable a historic Gitleaks scan. |
 

lib/gitlab/ci/parsers/security/common.rb

@@ -33,8 +33,7 @@ module Gitlab
             report_data
           rescue JSON::ParserError
             raise SecurityReportParserError, 'JSON parsing failed'
-          rescue StandardError => e
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+          rescue StandardError
             raise SecurityReportParserError, "#{report.type} security report parsing failed"
           end
 

lib/gitlab/database/reindexing/index_selection.rb

@@ -9,8 +9,8 @@ module Gitlab
         # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this
         MINIMUM_RELATIVE_BLOAT = 0.2
 
-        # Only consider indexes with a total ondisk size in this range (before reindexing)
-        INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze
+        # Only consider indexes beyond this size (before reindexing)
+        INDEX_SIZE_MINIMUM = 1.gigabyte
 
         delegate :each, to: :indexes
 
@@ -32,7 +32,7 @@ module Gitlab
 
           @indexes ||= candidates
             .not_recently_reindexed
-            .where(ondisk_size_bytes: INDEX_SIZE_RANGE)
+            .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM)
             .sort_by(&:relative_bloat_level) # forced N+1
             .reverse
             .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT }

lib/gitlab/database/reindexing/reindex_concurrently.rb

@@ -8,7 +8,7 @@ module Gitlab
         ReindexError = Class.new(StandardError)
 
         TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*'
-        STATEMENT_TIMEOUT = 9.hours
+        STATEMENT_TIMEOUT = 24.hours
         PG_MAX_INDEX_NAME_LENGTH = 63
 
         attr_reader :index, :logger

spec/lib/gitlab/database/reindexing/index_selection_spec.rb

@@ -46,14 +46,14 @@ RSpec.describe Gitlab::Database::Reindexing::IndexSelection do
     expect(subject).not_to include(excluded.index)
   end
 
-  it 'excludes indexes larger than 100 GB ondisk size' do
-    excluded = create(
+  it 'includes indexes larger than 100 GB ondisk size' do
+    included = create(
       :postgres_index_bloat_estimate,
       index: create(:postgres_index, ondisk_size_bytes: 101.gigabytes),
       bloat_size_bytes: 25.gigabyte
     )
 
-    expect(subject).not_to include(excluded.index)
+    expect(subject).to include(included.index)
   end
 
   context 'with time frozen' do

spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb

@@ -62,7 +62,7 @@ RSpec.describe Gitlab::Database::Reindexing::ReindexConcurrently, '#perform' do
 
   it 'recreates the index using REINDEX with a long statement timeout' do
     expect_to_execute_in_order(
-      "SET statement_timeout TO '32400s'",
+      "SET statement_timeout TO '86400s'",
       "REINDEX INDEX CONCURRENTLY \"public\".\"#{index.name}\"",
       "RESET statement_timeout"
     )
@@ -84,7 +84,7 @@ RSpec.describe Gitlab::Database::Reindexing::ReindexConcurrently, '#perform' do
       it 'drops the dangling indexes while controlling lock_timeout' do
         expect_to_execute_in_order(
           # Regular index rebuild
-          "SET statement_timeout TO '32400s'",
+          "SET statement_timeout TO '86400s'",
           "REINDEX INDEX CONCURRENTLY \"public\".\"#{index_name}\"",
           "RESET statement_timeout",
           # Drop _ccnew index

spec/migrations/add_open_source_plan_spec.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe AddOpenSourcePlan, :migration do
+  describe '#up' do
+    before do
+      allow(Gitlab).to receive(:dev_env_or_com?).and_return true
+    end
+
+    it 'creates 1 entry within the plans table' do
+      expect { migrate! }.to change { AddOpenSourcePlan::Plan.count }.by 1
+      expect(AddOpenSourcePlan::Plan.last.name).to eql('opensource')
+    end
+
+    it 'creates 1 entry for plan limits' do
+      expect { migrate! }.to change { AddOpenSourcePlan::PlanLimits.count }.by 1
+    end
+
+    context 'when the plan limits for gold and silver exists' do
+      before do
+        table(:plans).create!(id: 1, name: 'ultimate', title: 'Ultimate')
+        table(:plan_limits).create!(id: 1, plan_id: 1, storage_size_limit: 2000)
+      end
+
+      it 'duplicates the gold and silvers plan limits entries' do
+        migrate!
+
+        opensource_limits = AddOpenSourcePlan::Plan.find_by(name: 'opensource').limits
+        expect(opensource_limits.storage_size_limit).to be 2000
+      end
+    end
+
+    context 'when the instance is not SaaS' do
+      before do
+        allow(Gitlab).to receive(:dev_env_or_com?).and_return false
+      end
+
+      it 'does not create plans and plan limits and returns' do
+        expect { migrate! }.not_to change { AddOpenSourcePlan::Plan.count }
+      end
+    end
+  end
+
+  describe '#down' do
+    before do
+      table(:plans).create!(id: 3, name: 'other')
+      table(:plan_limits).create!(plan_id: 3)
+    end
+
+    context 'when the instance is SaaS' do
+      before do
+        allow(Gitlab).to receive(:dev_env_or_com?).and_return true
+      end
+
+      it 'removes the newly added opensource entry' do
+        migrate!
+
+        expect { described_class.new.down }.to change { AddOpenSourcePlan::Plan.count }.by(-1)
+        expect(AddOpenSourcePlan::Plan.find_by(name: 'opensource')).to be_nil
+
+        other_plan = AddOpenSourcePlan::Plan.find_by(name: 'other')
+        expect(other_plan).to be_persisted
+        expect(AddOpenSourcePlan::PlanLimits.count).to eq(1)
+        expect(AddOpenSourcePlan::PlanLimits.first.plan_id).to eq(other_plan.id)
+      end
+    end
+
+    context 'when the instance is not SaaS' do
+      before do
+        allow(Gitlab).to receive(:dev_env_or_com?).and_return false
+        table(:plans).create!(id: 1, name: 'opensource', title: 'Open Source Program')
+        table(:plan_limits).create!(id: 1, plan_id: 1)
+      end
+
+      it 'does not delete plans and plan limits and returns' do
+        migrate!
+
+        expect { described_class.new.down }.not_to change { AddOpenSourcePlan::Plan.count }
+        expect(AddOpenSourcePlan::PlanLimits.count).to eq(2)
+      end
+    end
+  end
+end

spec/models/loose_foreign_keys/deleted_record_spec.rb

@@ -24,26 +24,9 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do
     end
   end
 
-  describe '.mark_records_processed_for_table_between' do
-    it 'marks processed exactly one record' do
-      described_class.mark_records_processed_for_table_between(table, deleted_record_2, deleted_record_2)
-
-      expect(described_class.status_pending.count).to eq(3)
-      expect(described_class.status_processed.count).to eq(1)
-
-      processed_record = described_class.status_processed.first
-      expect(processed_record).to eq(deleted_record_2)
-    end
-
-    it 'deletes two records' do
-      described_class.mark_records_processed_for_table_between(table, deleted_record_2, deleted_record_4)
-
-      expect(described_class.status_pending.count).to eq(2)
-      expect(described_class.status_processed.count).to eq(2)
-    end
-
-    it 'deletes all records' do
-      described_class.mark_records_processed_for_table_between(table, deleted_record_1, deleted_record_4)
+  describe '.mark_records_processed' do
+    it 'updates all records' do
+      described_class.mark_records_processed([deleted_record_1, deleted_record_2, deleted_record_4])
 
       expect(described_class.status_pending.count).to eq(1)
       expect(described_class.status_processed.count).to eq(3)

spec/services/labels/transfer_service_spec.rb

@@ -3,107 +3,121 @@
 require 'spec_helper'
 
 RSpec.describe Labels::TransferService do
-  describe '#execute' do
-    let_it_be(:user) { create(:user) }
+  shared_examples 'transfer labels' do
+    describe '#execute' do
+      let_it_be(:user) { create(:user) }
 
-    let_it_be(:old_group_ancestor) { create(:group) }
-    let_it_be(:old_group) { create(:group, parent: old_group_ancestor) }
+      let_it_be(:old_group_ancestor) { create(:group) }
+      let_it_be(:old_group) { create(:group, parent: old_group_ancestor) }
 
-    let_it_be(:new_group) { create(:group) }
+      let_it_be(:new_group) { create(:group) }
 
-    let_it_be(:project) { create(:project, :repository, group: new_group) }
+      let_it_be(:project) { create(:project, :repository, group: new_group) }
 
-    subject(:service) { described_class.new(user, old_group, project) }
+      subject(:service) { described_class.new(user, old_group, project) }
 
-    before do
-      old_group_ancestor.add_developer(user)
-      new_group.add_developer(user)
-    end
+      before do
+        old_group_ancestor.add_developer(user)
+        new_group.add_developer(user)
+      end
 
-    it 'recreates missing group labels at project level and assigns them to the issuables' do
-      old_group_label_1 = create(:group_label, group: old_group)
-      old_group_label_2 = create(:group_label, group: old_group)
+      it 'recreates missing group labels at project level and assigns them to the issuables' do
+        old_group_label_1 = create(:group_label, group: old_group)
+        old_group_label_2 = create(:group_label, group: old_group)
 
-      labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label_1])
-      labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_label_2])
+        labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label_1])
+        labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_label_2])
 
-      expect { service.execute }.to change(project.labels, :count).by(2)
-      expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_1.title))
-      expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_2.title))
-    end
+        expect { service.execute }.to change(project.labels, :count).by(2)
+        expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_1.title))
+        expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_label_2.title))
+      end
 
-    it 'recreates missing ancestor group labels at project level and assigns them to the issuables' do
-      old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor)
-      old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor)
-
-      labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1])
-      labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2])
-
-      expect { service.execute }.to change(project.labels, :count).by(2)
-      expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_1.title))
-      expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_2.title))
-    end
-
-    it 'recreates label priorities related to the missing group labels' do
-      old_group_label = create(:group_label, group: old_group)
-      create(:labeled_issue, project: project, labels: [old_group_label])
-      create(:label_priority, project: project, label: old_group_label, priority: 1)
-
-      service.execute
-
-      new_project_label = project.labels.find_by(title: old_group_label.title)
-      expect(new_project_label.id).not_to eq old_group_label.id
-      expect(new_project_label.priorities).not_to be_empty
-    end
-
-    it 'does not recreate missing group labels that are not applied to issues or merge requests' do
-      old_group_label = create(:group_label, group: old_group)
-
-      service.execute
-
-      expect(project.labels.where(title: old_group_label.title)).to be_empty
-    end
-
-    it 'does not recreate missing group labels that already exist in the project group' do
-      old_group_label = create(:group_label, group: old_group)
-      labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label])
-
-      new_group_label = create(:group_label, group: new_group, title: old_group_label.title)
-
-      service.execute
-
-      expect(project.labels.where(title: old_group_label.title)).to be_empty
-      expect(labeled_issue.reload.labels).to contain_exactly(new_group_label)
-    end
-
-    it 'updates only label links in the given project' do
-      old_group_label = create(:group_label, group: old_group)
-      other_project = create(:project, group: old_group)
-
-      labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label])
-      other_project_labeled_issue = create(:labeled_issue, project: other_project, labels: [old_group_label])
-
-      service.execute
-
-      expect(labeled_issue.reload.labels).not_to include(old_group_label)
-      expect(other_project_labeled_issue.reload.labels).to contain_exactly(old_group_label)
-    end
-
-    context 'when moving within the same ancestor group' do
-      let(:other_subgroup) { create(:group, parent: old_group_ancestor) }
-      let(:project) { create(:project, :repository, group: other_subgroup) }
-
-      it 'does not recreate ancestor group labels' do
+      it 'recreates missing ancestor group labels at project level and assigns them to the issuables' do
         old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor)
         old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor)
 
         labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1])
         labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2])
 
-        expect { service.execute }.not_to change(project.labels, :count)
-        expect(labeled_issue.reload.labels).to contain_exactly(old_group_ancestor_label_1)
-        expect(labeled_merge_request.reload.labels).to contain_exactly(old_group_ancestor_label_2)
+        expect { service.execute }.to change(project.labels, :count).by(2)
+        expect(labeled_issue.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_1.title))
+        expect(labeled_merge_request.reload.labels).to contain_exactly(project.labels.find_by_title(old_group_ancestor_label_2.title))
+      end
+
+      it 'recreates label priorities related to the missing group labels' do
+        old_group_label = create(:group_label, group: old_group)
+        create(:labeled_issue, project: project, labels: [old_group_label])
+        create(:label_priority, project: project, label: old_group_label, priority: 1)
+
+        service.execute
+
+        new_project_label = project.labels.find_by(title: old_group_label.title)
+        expect(new_project_label.id).not_to eq old_group_label.id
+        expect(new_project_label.priorities).not_to be_empty
+      end
+
+      it 'does not recreate missing group labels that are not applied to issues or merge requests' do
+        old_group_label = create(:group_label, group: old_group)
+
+        service.execute
+
+        expect(project.labels.where(title: old_group_label.title)).to be_empty
+      end
+
+      it 'does not recreate missing group labels that already exist in the project group' do
+        old_group_label = create(:group_label, group: old_group)
+        labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label])
+
+        new_group_label = create(:group_label, group: new_group, title: old_group_label.title)
+
+        service.execute
+
+        expect(project.labels.where(title: old_group_label.title)).to be_empty
+        expect(labeled_issue.reload.labels).to contain_exactly(new_group_label)
+      end
+
+      it 'updates only label links in the given project' do
+        old_group_label = create(:group_label, group: old_group)
+        other_project = create(:project, group: old_group)
+
+        labeled_issue = create(:labeled_issue, project: project, labels: [old_group_label])
+        other_project_labeled_issue = create(:labeled_issue, project: other_project, labels: [old_group_label])
+
+        service.execute
+
+        expect(labeled_issue.reload.labels).not_to include(old_group_label)
+        expect(other_project_labeled_issue.reload.labels).to contain_exactly(old_group_label)
+      end
+
+      context 'when moving within the same ancestor group' do
+        let(:other_subgroup) { create(:group, parent: old_group_ancestor) }
+        let(:project) { create(:project, :repository, group: other_subgroup) }
+
+        it 'does not recreate ancestor group labels' do
+          old_group_ancestor_label_1 = create(:group_label, group: old_group_ancestor)
+          old_group_ancestor_label_2 = create(:group_label, group: old_group_ancestor)
+
+          labeled_issue = create(:labeled_issue, project: project, labels: [old_group_ancestor_label_1])
+          labeled_merge_request = create(:labeled_merge_request, source_project: project, labels: [old_group_ancestor_label_2])
+
+          expect { service.execute }.not_to change(project.labels, :count)
+          expect(labeled_issue.reload.labels).to contain_exactly(old_group_ancestor_label_1)
+          expect(labeled_merge_request.reload.labels).to contain_exactly(old_group_ancestor_label_2)
+        end
       end
     end
   end
+
+  context 'with use_optimized_group_labels_query FF on' do
+    it_behaves_like 'transfer labels'
+  end
+
+  context 'with use_optimized_group_labels_query FF off' do
+    before do
+      stub_feature_flags(use_optimized_group_labels_query: false)
+    end
+
+    it_behaves_like 'transfer labels'
+  end
 end