2016-10-27 12:59:52 +00:00
|
|
|
require 'fileutils'
|
|
|
|
|
2016-07-08 19:32:58 +00:00
|
|
|
require 'spec_helper'
|
|
|
|
|
|
|
|
describe GitGarbageCollectWorker do
|
2017-03-27 20:42:08 +00:00
|
|
|
let(:project) { create(:project, :repository) }
|
2016-07-08 19:32:58 +00:00
|
|
|
let(:shell) { Gitlab::Shell.new }
|
2017-09-04 17:55:04 +00:00
|
|
|
let!(:lease_uuid) { SecureRandom.uuid }
|
|
|
|
let!(:lease_key) { "project_housekeeping:#{project.id}" }
|
2016-07-08 19:32:58 +00:00
|
|
|
|
2017-05-01 15:13:33 +00:00
|
|
|
subject { described_class.new }
|
2016-07-08 19:32:58 +00:00
|
|
|
|
|
|
|
describe "#perform" do
|
2018-06-13 14:36:43 +00:00
|
|
|
context 'with active lease_uuid' do
|
|
|
|
before do
|
|
|
|
allow(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "flushes ref caches when the task if 'gc'" do
|
|
|
|
expect(subject).to receive(:renew_lease).with(lease_key, lease_uuid).and_call_original
|
|
|
|
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:garbage_collect)
|
|
|
|
.and_return(nil)
|
|
|
|
expect_any_instance_of(Repository).to receive(:after_create_branch).and_call_original
|
|
|
|
expect_any_instance_of(Repository).to receive(:branch_names).and_call_original
|
|
|
|
expect_any_instance_of(Repository).to receive(:has_visible_content?).and_call_original
|
|
|
|
expect_any_instance_of(Gitlab::Git::Repository).to receive(:has_visible_content?).and_call_original
|
|
|
|
|
|
|
|
subject.perform(project.id, :gc, lease_key, lease_uuid)
|
|
|
|
end
|
2018-07-10 22:08:54 +00:00
|
|
|
|
|
|
|
it 'handles gRPC errors' do
|
|
|
|
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:garbage_collect).and_raise(GRPC::NotFound)
|
|
|
|
|
|
|
|
expect { subject.perform(project.id, :gc, lease_key, lease_uuid) }.to raise_exception(Gitlab::Git::Repository::NoRepository)
|
|
|
|
end
|
2018-06-13 14:36:43 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'with different lease than the active one' do
|
|
|
|
before do
|
|
|
|
allow(subject).to receive(:get_lease_uuid).and_return(SecureRandom.uuid)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns silently' do
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:after_create_branch).and_call_original
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:branch_names).and_call_original
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:has_visible_content?).and_call_original
|
|
|
|
|
|
|
|
subject.perform(project.id, :gc, lease_key, lease_uuid)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'with no active lease' do
|
|
|
|
before do
|
|
|
|
allow(subject).to receive(:get_lease_uuid).and_return(false)
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when is able to get the lease' do
|
2017-09-04 17:55:04 +00:00
|
|
|
before do
|
2018-06-13 14:36:43 +00:00
|
|
|
allow(subject).to receive(:try_obtain_lease).and_return(SecureRandom.uuid)
|
2017-07-19 15:34:14 +00:00
|
|
|
end
|
2016-07-08 19:32:58 +00:00
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
it "flushes ref caches when the task if 'gc'" do
|
2018-06-13 14:36:43 +00:00
|
|
|
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:garbage_collect)
|
|
|
|
.and_return(nil)
|
2017-09-04 17:55:04 +00:00
|
|
|
expect_any_instance_of(Repository).to receive(:after_create_branch).and_call_original
|
|
|
|
expect_any_instance_of(Repository).to receive(:branch_names).and_call_original
|
2017-09-14 08:48:12 +00:00
|
|
|
expect_any_instance_of(Repository).to receive(:has_visible_content?).and_call_original
|
2017-09-29 13:08:44 +00:00
|
|
|
expect_any_instance_of(Gitlab::Git::Repository).to receive(:has_visible_content?).and_call_original
|
2017-09-04 17:55:04 +00:00
|
|
|
|
2018-06-13 14:36:43 +00:00
|
|
|
subject.perform(project.id)
|
2017-09-04 17:55:04 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-13 14:36:43 +00:00
|
|
|
context 'when no lease can be obtained' do
|
2017-09-04 17:55:04 +00:00
|
|
|
before do
|
2018-06-13 14:36:43 +00:00
|
|
|
expect(subject).to receive(:try_obtain_lease).and_return(false)
|
2017-09-04 17:55:04 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
it 'returns silently' do
|
|
|
|
expect(subject).not_to receive(:command)
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:after_create_branch).and_call_original
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:branch_names).and_call_original
|
|
|
|
expect_any_instance_of(Repository).not_to receive(:has_visible_content?).and_call_original
|
|
|
|
|
2018-06-13 14:36:43 +00:00
|
|
|
subject.perform(project.id)
|
2017-09-04 17:55:04 +00:00
|
|
|
end
|
|
|
|
end
|
2017-07-19 15:34:14 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
context "repack_full" do
|
2017-09-04 17:55:04 +00:00
|
|
|
before do
|
|
|
|
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
|
|
|
|
end
|
|
|
|
|
2017-07-19 15:34:14 +00:00
|
|
|
it "calls Gitaly" do
|
|
|
|
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:repack_full)
|
|
|
|
.and_return(nil)
|
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, :full_repack, lease_key, lease_uuid)
|
2017-07-19 15:34:14 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context "repack_incremental" do
|
2017-09-04 17:55:04 +00:00
|
|
|
before do
|
|
|
|
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
|
|
|
|
end
|
|
|
|
|
2017-07-19 15:34:14 +00:00
|
|
|
it "calls Gitaly" do
|
|
|
|
expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive(:repack_incremental)
|
|
|
|
.and_return(nil)
|
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, :incremental_repack, lease_key, lease_uuid)
|
2017-07-19 15:34:14 +00:00
|
|
|
end
|
2016-07-08 19:32:58 +00:00
|
|
|
end
|
2016-10-27 12:59:52 +00:00
|
|
|
|
|
|
|
shared_examples 'gc tasks' do
|
2017-06-14 18:18:56 +00:00
|
|
|
before do
|
2017-09-04 17:55:04 +00:00
|
|
|
allow(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
|
2017-06-14 18:18:56 +00:00
|
|
|
allow(subject).to receive(:bitmaps_enabled?).and_return(bitmaps_enabled)
|
|
|
|
end
|
2016-10-27 12:59:52 +00:00
|
|
|
|
|
|
|
it 'incremental repack adds a new packfile' do
|
|
|
|
create_objects(project)
|
|
|
|
before_packs = packs(project)
|
|
|
|
|
|
|
|
expect(before_packs.count).to be >= 1
|
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, 'incremental_repack', lease_key, lease_uuid)
|
2016-10-27 12:59:52 +00:00
|
|
|
after_packs = packs(project)
|
|
|
|
|
|
|
|
# Exactly one new pack should have been created
|
|
|
|
expect(after_packs.count).to eq(before_packs.count + 1)
|
|
|
|
|
|
|
|
# Previously existing packs are still around
|
|
|
|
expect(before_packs & after_packs).to eq(before_packs)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'full repack consolidates into 1 packfile' do
|
|
|
|
create_objects(project)
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, 'incremental_repack', lease_key, lease_uuid)
|
2016-10-27 12:59:52 +00:00
|
|
|
before_packs = packs(project)
|
|
|
|
|
|
|
|
expect(before_packs.count).to be >= 2
|
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, 'full_repack', lease_key, lease_uuid)
|
2016-10-27 12:59:52 +00:00
|
|
|
after_packs = packs(project)
|
|
|
|
|
|
|
|
expect(after_packs.count).to eq(1)
|
|
|
|
|
|
|
|
# Previously existing packs should be gone now
|
|
|
|
expect(after_packs - before_packs).to eq(after_packs)
|
|
|
|
|
|
|
|
expect(File.exist?(bitmap_path(after_packs.first))).to eq(bitmaps_enabled)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'gc consolidates into 1 packfile and updates packed-refs' do
|
|
|
|
create_objects(project)
|
|
|
|
before_packs = packs(project)
|
|
|
|
before_packed_refs = packed_refs(project)
|
|
|
|
|
|
|
|
expect(before_packs.count).to be >= 1
|
|
|
|
|
2017-09-04 17:55:04 +00:00
|
|
|
subject.perform(project.id, 'gc', lease_key, lease_uuid)
|
2016-10-27 12:59:52 +00:00
|
|
|
after_packed_refs = packed_refs(project)
|
|
|
|
after_packs = packs(project)
|
|
|
|
|
|
|
|
expect(after_packs.count).to eq(1)
|
|
|
|
|
|
|
|
# Previously existing packs should be gone now
|
|
|
|
expect(after_packs - before_packs).to eq(after_packs)
|
|
|
|
|
|
|
|
# The packed-refs file should have been updated during 'git gc'
|
|
|
|
expect(before_packed_refs).not_to eq(after_packed_refs)
|
|
|
|
|
|
|
|
expect(File.exist?(bitmap_path(after_packs.first))).to eq(bitmaps_enabled)
|
|
|
|
end
|
2018-03-04 06:18:32 +00:00
|
|
|
|
|
|
|
it 'cleans up repository after finishing' do
|
|
|
|
expect_any_instance_of(Project).to receive(:cleanup).and_call_original
|
|
|
|
|
|
|
|
subject.perform(project.id, 'gc', lease_key, lease_uuid)
|
|
|
|
end
|
2016-10-27 12:59:52 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'with bitmaps enabled' do
|
|
|
|
let(:bitmaps_enabled) { true }
|
|
|
|
|
|
|
|
include_examples 'gc tasks'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'with bitmaps disabled' do
|
|
|
|
let(:bitmaps_enabled) { false }
|
|
|
|
|
|
|
|
include_examples 'gc tasks'
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
# Create a new commit on a random new branch
|
|
|
|
def create_objects(project)
|
2018-06-05 15:58:28 +00:00
|
|
|
rugged = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
|
|
|
project.repository.rugged
|
|
|
|
end
|
2016-10-27 12:59:52 +00:00
|
|
|
old_commit = rugged.branches.first.target
|
|
|
|
new_commit_sha = Rugged::Commit.create(
|
|
|
|
rugged,
|
|
|
|
message: "hello world #{SecureRandom.hex(6)}",
|
2017-02-22 00:51:36 +00:00
|
|
|
author: Gitlab::Git.committer_hash(email: 'foo@bar', name: 'baz'),
|
|
|
|
committer: Gitlab::Git.committer_hash(email: 'foo@bar', name: 'baz'),
|
2016-10-27 12:59:52 +00:00
|
|
|
tree: old_commit.tree,
|
2017-05-03 11:27:17 +00:00
|
|
|
parents: [old_commit]
|
2016-10-27 12:59:52 +00:00
|
|
|
)
|
2017-09-01 10:11:59 +00:00
|
|
|
Gitlab::Git::OperationService.new(nil, project.repository.raw_repository).send(
|
2016-12-13 18:29:35 +00:00
|
|
|
:update_ref,
|
2016-10-27 12:59:52 +00:00
|
|
|
"refs/heads/#{SecureRandom.hex(6)}",
|
|
|
|
new_commit_sha,
|
|
|
|
Gitlab::Git::BLANK_SHA
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def packs(project)
|
2018-06-05 15:58:28 +00:00
|
|
|
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
|
|
|
|
Dir["#{project.repository.path_to_repo}/objects/pack/*.pack"]
|
|
|
|
end
|
2016-10-27 12:59:52 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def packed_refs(project)
|
|
|
|
path = "#{project.repository.path_to_repo}/packed-refs"
|
|
|
|
FileUtils.touch(path)
|
|
|
|
File.read(path)
|
|
|
|
end
|
|
|
|
|
|
|
|
def bitmap_path(pack)
|
|
|
|
pack.sub(/\.pack\z/, '.bitmap')
|
2016-07-08 19:32:58 +00:00
|
|
|
end
|
|
|
|
end
|