db759c5d9c
For a given merge request, it's quite common to see duplicate FindCommit Gitaly requests because the Gitaly CommitService caches the request by the commit SHA, not by the ref name. However, most of the duplicate requests use the ref name, so the cache is never actually used in practice. This leads to unnecessary requests that slow performance. This commit allows certain callers to bypass the ref name to OID conversion in the cache. We don't do this by default because it's possible the tip of the branch changes during the commit, which would cause the caller to get stale data. This commit also forces the Ci::Pipeline to use the full ref name so that caching can work for merge requests. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/57083
463 lines
17 KiB
Ruby
463 lines
17 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module GitalyClient
|
|
class CommitService
|
|
include Gitlab::EncodingHelper
|
|
|
|
def initialize(repository)
|
|
@gitaly_repo = repository.gitaly_repository
|
|
@repository = repository
|
|
end
|
|
|
|
def ls_files(revision)
|
|
request = Gitaly::ListFilesRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision)
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout)
|
|
response.flat_map do |msg|
|
|
msg.paths.map { |d| EncodingHelper.encode!(d.dup) }
|
|
end
|
|
end
|
|
|
|
def ancestor?(ancestor_id, child_id)
|
|
request = Gitaly::CommitIsAncestorRequest.new(
|
|
repository: @gitaly_repo,
|
|
ancestor_id: ancestor_id,
|
|
child_id: child_id
|
|
)
|
|
|
|
GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value
|
|
end
|
|
|
|
def diff(from, to, options = {})
|
|
from_id = case from
|
|
when NilClass
|
|
Gitlab::Git::EMPTY_TREE_ID
|
|
else
|
|
if from.respond_to?(:oid)
|
|
# This is meant to match a Rugged::Commit. This should be impossible in
|
|
# the future.
|
|
from.oid
|
|
else
|
|
from
|
|
end
|
|
end
|
|
|
|
to_id = case to
|
|
when NilClass
|
|
Gitlab::Git::EMPTY_TREE_ID
|
|
else
|
|
if to.respond_to?(:oid)
|
|
# This is meant to match a Rugged::Commit. This should be impossible in
|
|
# the future.
|
|
to.oid
|
|
else
|
|
to
|
|
end
|
|
end
|
|
|
|
request_params = diff_between_commits_request_params(from_id, to_id, options)
|
|
|
|
call_commit_diff(request_params, options)
|
|
end
|
|
|
|
def diff_from_parent(commit, options = {})
|
|
request_params = diff_from_parent_request_params(commit, options)
|
|
|
|
call_commit_diff(request_params, options)
|
|
end
|
|
|
|
def commit_deltas(commit)
|
|
request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit))
|
|
response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout)
|
|
|
|
response.flat_map { |msg| msg.deltas }
|
|
end
|
|
|
|
def tree_entry(ref, path, limit = nil)
|
|
if Pathname.new(path).cleanpath.to_s.start_with?('../')
|
|
# The TreeEntry RPC should return an empty reponse in this case but in
|
|
# Gitaly 0.107.0 and earlier we get an exception instead. This early return
|
|
# saves us a Gitaly roundtrip while also avoiding the exception.
|
|
return
|
|
end
|
|
|
|
request = Gitaly::TreeEntryRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(ref),
|
|
path: encode_binary(path),
|
|
limit: limit.to_i
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
entry = nil
|
|
data = []
|
|
response.each do |msg|
|
|
if entry.nil?
|
|
entry = msg
|
|
|
|
break unless entry.type == :BLOB
|
|
end
|
|
|
|
data << msg.data
|
|
end
|
|
entry.data = data.join
|
|
|
|
entry unless entry.oid.blank?
|
|
end
|
|
|
|
def tree_entries(repository, revision, path, recursive)
|
|
request = Gitaly::GetTreeEntriesRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision),
|
|
path: path.present? ? encode_binary(path) : '.',
|
|
recursive: recursive
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
response.flat_map do |message|
|
|
message.entries.map do |gitaly_tree_entry|
|
|
Gitlab::Git::Tree.new(
|
|
id: gitaly_tree_entry.oid,
|
|
root_id: gitaly_tree_entry.root_oid,
|
|
type: gitaly_tree_entry.type.downcase,
|
|
mode: gitaly_tree_entry.mode.to_s(8),
|
|
name: File.basename(gitaly_tree_entry.path),
|
|
path: encode_binary(gitaly_tree_entry.path),
|
|
flat_path: encode_binary(gitaly_tree_entry.flat_path),
|
|
commit_id: gitaly_tree_entry.commit_oid
|
|
)
|
|
end
|
|
end
|
|
end
|
|
|
|
def commit_count(ref, options = {})
|
|
request = Gitaly::CountCommitsRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(ref),
|
|
all: !!options[:all]
|
|
)
|
|
request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
|
|
request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
|
|
request.path = encode_binary(options[:path]) if options[:path].present?
|
|
request.max_count = options[:max_count] if options[:max_count].present?
|
|
|
|
GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count
|
|
end
|
|
|
|
def diverging_commit_count(from, to, max_count:)
|
|
request = Gitaly::CountDivergingCommitsRequest.new(
|
|
repository: @gitaly_repo,
|
|
from: encode_binary(from),
|
|
to: encode_binary(to),
|
|
max_count: max_count
|
|
)
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout)
|
|
[response.left_count, response.right_count]
|
|
end
|
|
|
|
def list_last_commits_for_tree(revision, path, offset: 0, limit: 25)
|
|
request = Gitaly::ListLastCommitsForTreeRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision),
|
|
path: encode_binary(path.to_s),
|
|
offset: offset,
|
|
limit: limit
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
response.each_with_object({}) do |gitaly_response, hsh|
|
|
gitaly_response.commits.each do |commit_for_tree|
|
|
hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit)
|
|
end
|
|
end
|
|
end
|
|
|
|
def last_commit_for_path(revision, path)
|
|
request = Gitaly::LastCommitForPathRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision),
|
|
path: encode_binary(path.to_s)
|
|
)
|
|
|
|
gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit
|
|
return unless gitaly_commit
|
|
|
|
Gitlab::Git::Commit.new(@repository, gitaly_commit)
|
|
end
|
|
|
|
def between(from, to)
|
|
request = Gitaly::CommitsBetweenRequest.new(
|
|
repository: @gitaly_repo,
|
|
from: from,
|
|
to: to
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout)
|
|
consume_commits_response(response)
|
|
end
|
|
|
|
def diff_stats(left_commit_sha, right_commit_sha)
|
|
request = Gitaly::DiffStatsRequest.new(
|
|
repository: @gitaly_repo,
|
|
left_commit_id: left_commit_sha,
|
|
right_commit_id: right_commit_sha
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout)
|
|
response.flat_map(&:stats)
|
|
end
|
|
|
|
def find_all_commits(opts = {})
|
|
request = Gitaly::FindAllCommitsRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: opts[:ref].to_s,
|
|
max_count: opts[:max_count].to_i,
|
|
skip: opts[:skip].to_i
|
|
)
|
|
request.order = opts[:order].upcase if opts[:order].present?
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout)
|
|
consume_commits_response(response)
|
|
end
|
|
|
|
def list_commits_by_oid(oids)
|
|
return [] if oids.empty?
|
|
|
|
request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
|
|
consume_commits_response(response)
|
|
rescue GRPC::NotFound # If no repository is found, happens mainly during testing
|
|
[]
|
|
end
|
|
|
|
def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0)
|
|
request = Gitaly::CommitsByMessageRequest.new(
|
|
repository: @gitaly_repo,
|
|
query: query,
|
|
revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT),
|
|
path: path.to_s.force_encoding(Encoding::ASCII_8BIT),
|
|
limit: limit.to_i,
|
|
offset: offset.to_i
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout)
|
|
consume_commits_response(response)
|
|
end
|
|
|
|
def languages(ref = nil)
|
|
request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
|
|
|
|
response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
|
|
end
|
|
|
|
def raw_blame(revision, path)
|
|
request = Gitaly::RawBlameRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision),
|
|
path: encode_binary(path)
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout)
|
|
response.reduce([]) { |memo, msg| memo << msg.data }.join
|
|
end
|
|
|
|
def find_commit(revision)
|
|
if Gitlab::SafeRequestStore.active?
|
|
# We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly
|
|
# because `revision` can be a branch name, so we can't use it as a key
|
|
# as it could point to another commit later on (happens a lot in
|
|
# tests).
|
|
key = {
|
|
storage: @gitaly_repo.storage_name,
|
|
relative_path: @gitaly_repo.relative_path,
|
|
commit_id: revision
|
|
}
|
|
return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key)
|
|
|
|
commit = call_find_commit(revision)
|
|
return unless commit
|
|
|
|
key[:commit_id] = commit.id unless GitalyClient.ref_name_caching_allowed?
|
|
Gitlab::SafeRequestStore[key] = commit
|
|
else
|
|
call_find_commit(revision)
|
|
end
|
|
end
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
def patch(revision)
|
|
request = Gitaly::CommitPatchRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision)
|
|
)
|
|
response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
response.sum(&:data)
|
|
end
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
def commit_stats(revision)
|
|
request = Gitaly::CommitStatsRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision)
|
|
)
|
|
GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout)
|
|
end
|
|
|
|
def find_commits(options)
|
|
request = Gitaly::FindCommitsRequest.new(
|
|
repository: @gitaly_repo,
|
|
limit: options[:limit],
|
|
offset: options[:offset],
|
|
follow: options[:follow],
|
|
skip_merges: options[:skip_merges],
|
|
all: !!options[:all],
|
|
disable_walk: true # This option is deprecated. The 'walk' implementation is being removed.
|
|
)
|
|
request.after = GitalyClient.timestamp(options[:after]) if options[:after]
|
|
request.before = GitalyClient.timestamp(options[:before]) if options[:before]
|
|
request.revision = encode_binary(options[:ref]) if options[:ref]
|
|
|
|
request.paths = encode_repeated(Array(options[:path])) if options[:path].present?
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
consume_commits_response(response)
|
|
end
|
|
|
|
def filter_shas_with_signatures(shas)
|
|
request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo)
|
|
|
|
enum = Enumerator.new do |y|
|
|
shas.each_slice(20) do |revs|
|
|
request.shas = encode_repeated(revs)
|
|
|
|
y.yield request
|
|
|
|
request = Gitaly::FilterShasWithSignaturesRequest.new
|
|
end
|
|
end
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout)
|
|
|
|
response.flat_map do |msg|
|
|
msg.shas.map { |sha| EncodingHelper.encode!(sha) }
|
|
end
|
|
end
|
|
|
|
def extract_signature(commit_id)
|
|
request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
|
|
|
|
signature = +''.b
|
|
signed_text = +''.b
|
|
|
|
response.each do |message|
|
|
signature << message.signature
|
|
signed_text << message.signed_text
|
|
end
|
|
|
|
return if signature.blank? && signed_text.blank?
|
|
|
|
[signature, signed_text]
|
|
rescue GRPC::InvalidArgument => ex
|
|
raise ArgumentError, ex
|
|
end
|
|
|
|
def get_commit_signatures(commit_ids)
|
|
request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout)
|
|
|
|
signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] }
|
|
current_commit_id = nil
|
|
|
|
response.each do |message|
|
|
current_commit_id = message.commit_id if message.commit_id.present?
|
|
|
|
signatures[current_commit_id].first << message.signature
|
|
signatures[current_commit_id].last << message.signed_text
|
|
end
|
|
|
|
signatures
|
|
rescue GRPC::InvalidArgument => ex
|
|
raise ArgumentError, ex
|
|
end
|
|
|
|
def get_commit_messages(commit_ids)
|
|
request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout)
|
|
|
|
messages = Hash.new { |h, k| h[k] = +''.b }
|
|
current_commit_id = nil
|
|
|
|
response.each do |rpc_message|
|
|
current_commit_id = rpc_message.commit_id if rpc_message.commit_id.present?
|
|
|
|
messages[current_commit_id] << rpc_message.message
|
|
end
|
|
|
|
messages
|
|
end
|
|
|
|
private
|
|
|
|
def call_commit_diff(request_params, options = {})
|
|
request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
|
|
request_params[:enforce_limits] = options.fetch(:limits, true)
|
|
request_params[:collapse_diffs] = !options.fetch(:expanded, true)
|
|
request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h)
|
|
|
|
request = Gitaly::CommitDiffRequest.new(request_params)
|
|
response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout)
|
|
GitalyClient::DiffStitcher.new(response)
|
|
end
|
|
|
|
def diff_from_parent_request_params(commit, options = {})
|
|
parent_id = commit.parent_ids.first || Gitlab::Git::EMPTY_TREE_ID
|
|
|
|
diff_between_commits_request_params(parent_id, commit.id, options)
|
|
end
|
|
|
|
def diff_between_commits_request_params(from_id, to_id, options)
|
|
{
|
|
repository: @gitaly_repo,
|
|
left_commit_id: from_id,
|
|
right_commit_id: to_id,
|
|
paths: options.fetch(:paths, []).compact.map { |path| encode_binary(path) }
|
|
}
|
|
end
|
|
|
|
def consume_commits_response(response)
|
|
response.flat_map do |message|
|
|
message.commits.map do |gitaly_commit|
|
|
Gitlab::Git::Commit.new(@repository, gitaly_commit)
|
|
end
|
|
end
|
|
end
|
|
|
|
def encode_repeated(array)
|
|
Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } )
|
|
end
|
|
|
|
def call_find_commit(revision)
|
|
request = Gitaly::FindCommitRequest.new(
|
|
repository: @gitaly_repo,
|
|
revision: encode_binary(revision)
|
|
)
|
|
|
|
response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout)
|
|
|
|
response.commit
|
|
end
|
|
end
|
|
end
|
|
end
|