Use git_garbage_collect_worker to run pack_refs

PackRefs is not an expensive gitaly call - we want to
call it more often (than as part of full `gc`) because
it helps to keep number of refs files small - too many
refs file may be a problem for deployments with
slow storage.
This commit is contained in:
Jan Provaznik 2019-05-02 21:41:05 +00:00 committed by Stan Hu
parent 8163e23325
commit d25239ee0b
6 changed files with 47 additions and 4 deletions

View file

@ -11,6 +11,7 @@ module Projects
class HousekeepingService < BaseService
# Timeout set to 24h
LEASE_TIMEOUT = 86400
PACK_REFS_PERIOD = 6
class LeaseTaken < StandardError
def to_s
@ -76,13 +77,15 @@ module Projects
:gc
elsif pushes_since_gc % full_repack_period == 0
:full_repack
else
elsif pushes_since_gc % repack_period == 0
:incremental_repack
else
:pack_refs
end
end
def period_match?
[gc_period, full_repack_period, repack_period].any? { |period| pushes_since_gc % period == 0 }
[gc_period, full_repack_period, repack_period, PACK_REFS_PERIOD].any? { |period| pushes_since_gc % period == 0 }
end
def housekeeping_enabled?

View file

@ -29,7 +29,7 @@ class GitGarbageCollectWorker
# Refresh the branch cache in case garbage collection caused a ref lookup to fail
flush_ref_caches(project) if task == :gc
project.repository.expire_statistics_caches
project.repository.expire_statistics_caches if task != :pack_refs
# In case pack files are deleted, release libgit2 cache and open file
# descriptors ASAP instead of waiting for Ruby garbage collection
@ -58,7 +58,12 @@ class GitGarbageCollectWorker
## `repository` has to be a Gitlab::Git::Repository
def gitaly_call(task, repository)
client = Gitlab::GitalyClient::RepositoryService.new(repository)
client = if task == :pack_refs
Gitlab::GitalyClient::RefService.new(repository)
else
Gitlab::GitalyClient::RepositoryService.new(repository)
end
case task
when :gc
client.garbage_collect(bitmaps_enabled?)
@ -66,6 +71,8 @@ class GitGarbageCollectWorker
client.repack_full(bitmaps_enabled?)
when :incremental_repack
client.repack_incremental
when :pack_refs
client.pack_refs
end
rescue GRPC::NotFound => e
Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")

View file

@ -239,6 +239,12 @@ module Gitlab
messages
end
def pack_refs
request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
GitalyClient.call(@storage, :ref_service, :pack_refs, request)
end
private
def consume_refs_response(response)

View file

@ -165,4 +165,15 @@ describe Gitlab::GitalyClient::RefService do
client.delete_refs(except_with_prefixes: prefixes)
end
end
describe '#pack_refs' do
it 'sends a pack_refs message' do
expect_any_instance_of(Gitaly::RefService::Stub)
.to receive(:pack_refs)
.with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
.and_return(double(:pack_refs_response))
client.pack_refs
end
end
end

View file

@ -81,6 +81,9 @@ describe Projects::HousekeepingService do
# At push 10, 20, ... (except those above)
expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id, :incremental_repack, :the_lease_key, :the_uuid)
.exactly(16).times
# At push 6, 12, 18, ... (except those above)
expect(GitGarbageCollectWorker).to receive(:perform_async).with(project.id, :pack_refs, :the_lease_key, :the_uuid)
.exactly(27).times
201.times do
subject.increment!

View file

@ -115,6 +115,19 @@ describe GitGarbageCollectWorker do
end
end
context "pack_refs" do
before do
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)
end
it "calls Gitaly" do
expect_any_instance_of(Gitlab::GitalyClient::RefService).to receive(:pack_refs)
.and_return(nil)
subject.perform(project.id, :pack_refs, lease_key, lease_uuid)
end
end
context "repack_incremental" do
before do
expect(subject).to receive(:get_lease_uuid).and_return(lease_uuid)