diff --git a/changelogs/unreleased/add-new-arg-to-git-rev-list-call.yml b/changelogs/unreleased/add-new-arg-to-git-rev-list-call.yml new file mode 100644 index 00000000000..86680b6b117 --- /dev/null +++ b/changelogs/unreleased/add-new-arg-to-git-rev-list-call.yml @@ -0,0 +1,5 @@ +--- +title: Improve performance of LFS integrity check +merge_request: 19494 +author: +type: performance diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index b230289e7bf..f3cc388ea41 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -30,7 +30,7 @@ module Gitlab def git_new_pointers(object_limit, not_in) @new_pointers ||= begin - rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| + rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids| object_ids = object_ids.take(object_limit) if object_limit Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) @@ -39,13 +39,12 @@ module Gitlab end def git_all_pointers - params = { require_path: true } - - if Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0') + params = {} + if rev_list_supports_new_options? params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"] end - rev_list.all_objects(params) do |object_ids| + rev_list.all_objects(rev_list_params(params)) do |object_ids| Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) end end @@ -53,6 +52,23 @@ module Gitlab def rev_list Gitlab::Git::RevList.new(@repository, newrev: @newrev) end + + # We're passing the `--in-commit-order` arg to ensure we don't wait + # for git to traverse all commits before returning pointers. + # This is required in order to improve the performance of LFS integrity check + def rev_list_params(params = {}) + params[:options] ||= [] + params[:options] << "--in-commit-order" if rev_list_supports_new_options? + params[:require_path] = true + + params + end + + def rev_list_supports_new_options? + return @option_supported if defined?(@option_supported) + + @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0') + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 33b641a090b..4e661eceffb 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -27,9 +27,10 @@ module Gitlab # # When given a block it will yield objects as a lazy enumerator so # the caller can limit work done instead of processing megabytes of data - def new_objects(require_path: nil, not_in: nil, &lazy_block) + def new_objects(options: [], require_path: nil, not_in: nil, &lazy_block) opts = { including: newrev, + options: options, excluding: not_in.nil? ? :all : not_in, require_path: require_path }