diff --git a/.gitlab/ci/glfm.gitlab-ci.yml b/.gitlab/ci/glfm.gitlab-ci.yml deleted file mode 100644 index fcc9a035c1f..00000000000 --- a/.gitlab/ci/glfm.gitlab-ci.yml +++ /dev/null @@ -1,12 +0,0 @@ -glfm-verify: - # NOTE: We do not restrict this job to any specific subset of file changes via rules, because - # there are potentially many different source files within the codebase which could - # change the contents of the generated GLFM files. It is therefore safer to always - # run this job to ensure that no changes are missed. - extends: - - .rspec-ee-base-pg12 - stage: test - needs: ["setup-test-env"] - script: - - !reference [.base-script, script] - - bundle exec scripts/glfm/verify-all-generated-files-are-up-to-date.rb diff --git a/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb new file mode 100644 index 00000000000..16ce9bd5c85 --- /dev/null +++ b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class ScheduleBackfillNamespaceDetails < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillNamespaceDetails' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 200 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :namespaces, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) + end +end diff --git a/db/schema_migrations/20220921093355 b/db/schema_migrations/20220921093355 new file mode 100644 index 00000000000..6083a370279 --- /dev/null +++ b/db/schema_migrations/20220921093355 @@ -0,0 +1 @@ +d7640b3756cf8cddf9f798362d6d8445a1d37092e4dff9ae263fe39a661b1e55 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb new file mode 100644 index 00000000000..b8a51b576b6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill namespace_details for a range of namespaces + class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + upsert_namespace_details(sub_batch) + end + end + + def upsert_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id + FROM namespaces + WHERE namespaces.id IN(#{relation.select(:id).to_sql}) + AND namespaces.type <> 'Project' + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb new file mode 100644 index 00000000000..b6282de0da6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration do + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + + subject(:perform_migration) do + described_class.new(start_id: namespaces.minimum(:id), + end_id: namespaces.maximum(:id), + batch_table: :namespaces, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + describe '#perform' do + it 'creates details for all namespaces in range' do + namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1", + description_html: "Some description html1", cached_markdown_version: 4) + namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project', + description: "Some description2", description_html: "Some description html2", + cached_markdown_version: 4) + namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3", + description_html: "Some description html3", cached_markdown_version: 4) + namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3", + description_html: "Some description html4", cached_markdown_version: 4) + namespace_details.delete_all + + expect(namespace_details.pluck(:namespace_id)).to eql [] + + expect { perform_migration } + .to change { namespace_details.pluck(:namespace_id) }.from([]).to contain_exactly( + namespace1.id, + namespace3.id, + namespace4.id + ) + + expect(namespace_details.find_by_namespace_id(namespace1.id)).to have_attributes(migrated_attributes(namespace1)) + expect(namespace_details.find_by_namespace_id(namespace3.id)).to have_attributes(migrated_attributes(namespace3)) + expect(namespace_details.find_by_namespace_id(namespace4.id)).to have_attributes(migrated_attributes(namespace4)) + end + end + + def migrated_attributes(namespace) + { + description: namespace.description, + description_html: namespace.description_html, + cached_markdown_version: namespace.cached_markdown_version + } + end +end diff --git a/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb b/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb new file mode 100644 index 00000000000..61e4af3d10c --- /dev/null +++ b/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe ScheduleBackfillNamespaceDetails, schema: 20220921093355 do + context 'when on gitlab.com' do + let(:background_migration) { described_class::MIGRATION } + let(:migration) { described_class.new } + + before do + migration.up + end + + describe '#up' do + it 'schedules background jobs for each batch of projects' do + expect(background_migration).to( + have_scheduled_batched_migration( + table_name: :namespaces, + column_name: :id, + interval: described_class::INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + ) + end + end + + describe '#down' do + it 'deletes all batched migration records' do + migration.down + + expect(described_class::MIGRATION).not_to have_scheduled_batched_migration + end + end + end +end diff --git a/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb b/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb index d7a5bc6cc41..a4db4e25db3 100644 --- a/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.shared_examples 'a cascading namespace setting boolean attribute' do - |settings_association: :namespace_settings, settings_attribute_name:| + |settings_attribute_name:, settings_association: :namespace_settings| let_it_be_with_reload(:group) { create(:group) } let_it_be_with_reload(:subgroup) { create(:group, parent: group) } let(:group_settings) { group.send(settings_association) }