diff --git a/app/workers/namespaceless_project_destroy_worker.rb b/app/workers/namespaceless_project_destroy_worker.rb new file mode 100644 index 00000000000..bfae0c77700 --- /dev/null +++ b/app/workers/namespaceless_project_destroy_worker.rb @@ -0,0 +1,43 @@ +# Worker to destroy projects that do not have a namespace +# +# It destroys everything it can without having the info about the namespace it +# used to belong to. Projects in this state should be rare. +# The worker will reject doing anything for projects that *do* have a +# namespace. For those use ProjectDestroyWorker instead. +class NamespacelessProjectDestroyWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def self.bulk_perform_async(args_list) + Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list) + end + + def perform(project_id) + begin + project = Project.unscoped.find(project_id) + rescue ActiveRecord::RecordNotFound + return + end + return unless project.namespace_id.nil? # Reject doing anything for projects that *do* have a namespace + + project.team.truncate + + unlink_fork(project) if project.forked? + + # Override Project#remove_pages for this instance so it doesn't do anything + def project.remove_pages + end + + project.destroy! + end + + private + + def unlink_fork(project) + merge_requests = project.forked_from_project.merge_requests.opened.from_project(project) + + merge_requests.update_all(state: 'closed') + + project.forked_project_link.destroy + end +end diff --git a/changelogs/unreleased/tc-clean-pending-delete-projects.yml b/changelogs/unreleased/tc-clean-pending-delete-projects.yml new file mode 100644 index 00000000000..31b43999c31 --- /dev/null +++ b/changelogs/unreleased/tc-clean-pending-delete-projects.yml @@ -0,0 +1,4 @@ +--- +title: Add post-deploy migration to clean up projects in `pending_delete` state +merge_request: 11044 +author: diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 433381e79d3..0ca1f565185 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -40,6 +40,7 @@ - [expire_build_instance_artifacts, 1] - [group_destroy, 1] - [irker, 1] + - [namespaceless_project_destroy, 1] - [project_cache, 1] - [project_destroy, 1] - [project_export, 1] diff --git a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb new file mode 100644 index 00000000000..2d242da9ef8 --- /dev/null +++ b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb @@ -0,0 +1,47 @@ +# This is the counterpart of RequeuePendingDeleteProjects and cleans all +# projects with `pending_delete = true` and that do not have a namespace. +class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + @offset = 0 + + loop do + ids = pending_delete_batch + + break if ids.empty? + + args = ids.map { |id| Array(id) } + + NamespacelessProjectDestroyWorker.bulk_perform_async(args) + + @offset += 1 + end + end + + def down + # noop + end + + private + + def pending_delete_batch + connection.exec_query(find_batch).map{ |row| row['id'] } + end + + BATCH_SIZE = 5000 + + def find_batch + projects = Arel::Table.new(:projects) + projects.project(projects[:id]). + where(projects[:pending_delete].eq(true)). + where(projects[:namespace_id].eq(nil)). + skip(@offset * BATCH_SIZE). + take(BATCH_SIZE). + to_sql + end +end diff --git a/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb b/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb new file mode 100644 index 00000000000..0b8af5010ba --- /dev/null +++ b/spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170502101023_cleanup_namespaceless_pending_delete_projects.rb') + +describe CleanupNamespacelessPendingDeleteProjects do + before do + # Stub after_save callbacks that will fail when Project has no namespace + allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil) + allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil) + end + + describe '#up' do + it 'only cleans up pending delete projects' do + create(:empty_project) + create(:empty_project, pending_delete: true) + project = build(:empty_project, pending_delete: true, namespace_id: nil) + project.save(validate: false) + + expect(NamespacelessProjectDestroyWorker).to receive(:bulk_perform_async).with([[project.id.to_s]]) + + described_class.new.up + end + + it 'does nothing when no pending delete projects without namespace found' do + create(:empty_project) + create(:empty_project, pending_delete: true) + + expect(NamespacelessProjectDestroyWorker).not_to receive(:bulk_perform_async) + + described_class.new.up + end + end +end diff --git a/spec/workers/namespaceless_project_destroy_worker_spec.rb b/spec/workers/namespaceless_project_destroy_worker_spec.rb new file mode 100644 index 00000000000..8533b7b85e9 --- /dev/null +++ b/spec/workers/namespaceless_project_destroy_worker_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +describe NamespacelessProjectDestroyWorker do + subject { described_class.new } + + before do + # Stub after_save callbacks that will fail when Project has no namespace + allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil) + allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil) + end + + describe '#perform' do + context 'project has namespace' do + it 'does not do anything' do + project = create(:empty_project) + + subject.perform(project.id) + + expect(Project.unscoped.all).to include(project) + end + end + + context 'project has no namespace' do + let!(:project) do + project = build(:empty_project, namespace_id: nil) + project.save(validate: false) + project + end + + context 'project not a fork of another project' do + it "truncates the project's team" do + expect_any_instance_of(ProjectTeam).to receive(:truncate) + + subject.perform(project.id) + end + + it 'deletes the project' do + subject.perform(project.id) + + expect(Project.unscoped.all).not_to include(project) + end + + it 'does not call unlink_fork' do + is_expected.not_to receive(:unlink_fork) + + subject.perform(project.id) + end + + it 'does not do anything in Project#remove_pages method' do + expect(Gitlab::PagesTransfer).not_to receive(:new) + + subject.perform(project.id) + end + end + + context 'project forked from another' do + let!(:parent_project) { create(:empty_project) } + + before do + create(:forked_project_link, forked_to_project: project, forked_from_project: parent_project) + end + + it 'closes open merge requests' do + merge_request = create(:merge_request, source_project: project, target_project: parent_project) + + subject.perform(project.id) + + expect(merge_request.reload).to be_closed + end + + it 'destroys the link' do + subject.perform(project.id) + + expect(parent_project.forked_project_links).to be_empty + end + end + end + end +end