Merge branch 'zj-gitaly-pipelines-n-1' into 'master'
Improve performance of Pipelines#index.json See merge request gitlab-org/gitlab-ce!14846
This commit is contained in:
commit
ac22576d80
2
Gemfile
2
Gemfile
|
@ -263,7 +263,7 @@ gem 'gettext_i18n_rails', '~> 1.8.0'
|
|||
gem 'gettext_i18n_rails_js', '~> 1.2.0'
|
||||
gem 'gettext', '~> 3.2.2', require: false, group: :development
|
||||
|
||||
gem 'batch-loader'
|
||||
gem 'batch-loader', '~> 1.2.1'
|
||||
|
||||
# Perf bar
|
||||
gem 'peek', '~> 1.0.1'
|
||||
|
|
|
@ -78,7 +78,7 @@ GEM
|
|||
thread_safe (~> 0.3, >= 0.3.1)
|
||||
babosa (1.0.2)
|
||||
base32 (0.3.2)
|
||||
batch-loader (1.1.1)
|
||||
batch-loader (1.2.1)
|
||||
bcrypt (3.1.11)
|
||||
bcrypt_pbkdf (1.0.0)
|
||||
benchmark-ips (2.3.0)
|
||||
|
@ -988,7 +988,7 @@ DEPENDENCIES
|
|||
awesome_print (~> 1.2.0)
|
||||
babosa (~> 1.0.2)
|
||||
base32 (~> 0.3.0)
|
||||
batch-loader
|
||||
batch-loader (~> 1.2.1)
|
||||
bcrypt_pbkdf (~> 1.0)
|
||||
benchmark-ips (~> 2.3.0)
|
||||
better_errors (~> 2.1.0)
|
||||
|
|
|
@ -29,6 +29,8 @@ class Projects::PipelinesController < Projects::ApplicationController
|
|||
@pipelines_count = PipelinesFinder
|
||||
.new(project).execute.count
|
||||
|
||||
@pipelines.map(&:commit) # List commits for batch loading
|
||||
|
||||
respond_to do |format|
|
||||
format.html
|
||||
format.json do
|
||||
|
|
|
@ -287,8 +287,12 @@ module Ci
|
|||
Ci::Pipeline.truncate_sha(sha)
|
||||
end
|
||||
|
||||
# NOTE: This is loaded lazily and will never be nil, even if the commit
|
||||
# cannot be found.
|
||||
#
|
||||
# Use constructs like: `pipeline.commit.present?`
|
||||
def commit
|
||||
@commit ||= project.commit_by(oid: sha)
|
||||
@commit ||= Commit.lazy(project, sha)
|
||||
end
|
||||
|
||||
def branch?
|
||||
|
@ -338,12 +342,9 @@ module Ci
|
|||
end
|
||||
|
||||
def latest?
|
||||
return false unless ref
|
||||
return false unless ref && commit.present?
|
||||
|
||||
commit = project.commit(ref)
|
||||
return false unless commit
|
||||
|
||||
commit.sha == sha
|
||||
project.commit(ref) == commit
|
||||
end
|
||||
|
||||
def retried
|
||||
|
|
|
@ -86,6 +86,20 @@ class Commit
|
|||
def valid_hash?(key)
|
||||
!!(/\A#{COMMIT_SHA_PATTERN}\z/ =~ key)
|
||||
end
|
||||
|
||||
def lazy(project, oid)
|
||||
BatchLoader.for({ project: project, oid: oid }).batch do |items, loader|
|
||||
items_by_project = items.group_by { |i| i[:project] }
|
||||
|
||||
items_by_project.each do |project, commit_ids|
|
||||
oids = commit_ids.map { |i| i[:oid] }
|
||||
|
||||
project.repository.commits_by(oids: oids).each do |commit|
|
||||
loader.call({ project: commit.project, oid: commit.id }, commit) if commit
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
attr_accessor :raw
|
||||
|
@ -103,7 +117,7 @@ class Commit
|
|||
end
|
||||
|
||||
def ==(other)
|
||||
(self.class === other) && (raw == other.raw)
|
||||
other.is_a?(self.class) && raw == other.raw
|
||||
end
|
||||
|
||||
def self.reference_prefix
|
||||
|
@ -224,8 +238,8 @@ class Commit
|
|||
notes.includes(:author)
|
||||
end
|
||||
|
||||
def method_missing(m, *args, &block)
|
||||
@raw.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
|
||||
def method_missing(method, *args, &block)
|
||||
@raw.__send__(method, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
|
||||
end
|
||||
|
||||
def respond_to_missing?(method, include_private = false)
|
||||
|
|
|
@ -118,6 +118,18 @@ class Repository
|
|||
@commit_cache[oid] = find_commit(oid)
|
||||
end
|
||||
|
||||
def commits_by(oids:)
|
||||
return [] unless oids.present?
|
||||
|
||||
commits = Gitlab::Git::Commit.batch_by_oid(raw_repository, oids)
|
||||
|
||||
if commits.present?
|
||||
Commit.decorate(commits, @project)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
def commits(ref, path: nil, limit: nil, offset: nil, skip_merges: false, after: nil, before: nil)
|
||||
options = {
|
||||
repo: raw_repository,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
#js-pipeline-header-vue.pipeline-header-container
|
||||
|
||||
- if @commit
|
||||
- if @commit.present?
|
||||
.commit-box
|
||||
%h3.commit-title
|
||||
= markdown(@commit.title, pipeline: :single_line)
|
||||
|
|
|
@ -13,7 +13,7 @@ class ExpirePipelineCacheWorker
|
|||
|
||||
store.touch(project_pipelines_path(project))
|
||||
store.touch(project_pipeline_path(project, pipeline))
|
||||
store.touch(commit_pipelines_path(project, pipeline.commit)) if pipeline.commit
|
||||
store.touch(commit_pipelines_path(project, pipeline.commit)) unless pipeline.commit.nil?
|
||||
store.touch(new_merge_request_pipelines_path(project))
|
||||
each_pipelines_merge_request_path(project, pipeline) do |path|
|
||||
store.touch(path)
|
||||
|
|
|
@ -228,6 +228,19 @@ module Gitlab
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Only to be used when the object ids will not necessarily have a
|
||||
# relation to each other. The last 10 commits for a branch for example,
|
||||
# should go through .where
|
||||
def batch_by_oid(repo, oids)
|
||||
repo.gitaly_migrate(:list_commits_by_oid) do |is_enabled|
|
||||
if is_enabled
|
||||
repo.gitaly_commit_client.list_commits_by_oid(oids)
|
||||
else
|
||||
oids.map { |oid| find(repo, oid) }.compact
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(repository, raw_commit, head = nil)
|
||||
|
|
|
@ -169,6 +169,15 @@ module Gitlab
|
|||
consume_commits_response(response)
|
||||
end
|
||||
|
||||
def list_commits_by_oid(oids)
|
||||
request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids)
|
||||
|
||||
response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
|
||||
consume_commits_response(response)
|
||||
rescue GRPC::Unknown # If no repository is found, happens mainly during testing
|
||||
[]
|
||||
end
|
||||
|
||||
def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0)
|
||||
request = Gitaly::CommitsByMessageRequest.new(
|
||||
repository: @gitaly_repo,
|
||||
|
|
|
@ -17,13 +17,10 @@ describe Projects::PipelinesController do
|
|||
|
||||
describe 'GET index.json' do
|
||||
before do
|
||||
branch_head = project.commit
|
||||
parent = branch_head.parent
|
||||
|
||||
create(:ci_empty_pipeline, status: 'pending', project: project, sha: branch_head.id)
|
||||
create(:ci_empty_pipeline, status: 'running', project: project, sha: branch_head.id)
|
||||
create(:ci_empty_pipeline, status: 'created', project: project, sha: parent.id)
|
||||
create(:ci_empty_pipeline, status: 'success', project: project, sha: parent.id)
|
||||
%w(pending running created success).each_with_index do |status, index|
|
||||
sha = project.commit("HEAD~#{index}")
|
||||
create(:ci_empty_pipeline, status: status, project: project, sha: sha)
|
||||
end
|
||||
end
|
||||
|
||||
subject do
|
||||
|
@ -46,7 +43,7 @@ describe Projects::PipelinesController do
|
|||
|
||||
context 'when performing gitaly calls', :request_store do
|
||||
it 'limits the Gitaly requests' do
|
||||
expect { subject }.to change { Gitlab::GitalyClient.get_request_count }.by(8)
|
||||
expect { subject }.to change { Gitlab::GitalyClient.get_request_count }.by(3)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -13,6 +13,45 @@ describe Commit do
|
|||
it { is_expected.to include_module(StaticModel) }
|
||||
end
|
||||
|
||||
describe '.lazy' do
|
||||
set(:project) { create(:project, :repository) }
|
||||
|
||||
context 'when the commits are found' do
|
||||
let(:oids) do
|
||||
%w(
|
||||
498214de67004b1da3d820901307bed2a68a8ef6
|
||||
c642fe9b8b9f28f9225d7ea953fe14e74748d53b
|
||||
6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9
|
||||
048721d90c449b244b7b4c53a9186b04330174ec
|
||||
281d3a76f31c812dbf48abce82ccf6860adedd81
|
||||
)
|
||||
end
|
||||
|
||||
subject { oids.map { |oid| described_class.lazy(project, oid) } }
|
||||
|
||||
it 'batches requests for commits' do
|
||||
expect(project.repository).to receive(:commits_by).once.and_call_original
|
||||
|
||||
subject.first.title
|
||||
subject.last.title
|
||||
end
|
||||
|
||||
it 'maintains ordering' do
|
||||
subject.each_with_index do |commit, i|
|
||||
expect(commit.id).to eq(oids[i])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when not found' do
|
||||
it 'returns nil as commit' do
|
||||
commit = described_class.lazy(project, 'deadbeef').__sync
|
||||
|
||||
expect(commit).to be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#author' do
|
||||
it 'looks up the author in a case-insensitive way' do
|
||||
user = create(:user, email: commit.author_email.upcase)
|
||||
|
|
|
@ -239,6 +239,54 @@ describe Repository do
|
|||
end
|
||||
end
|
||||
|
||||
describe '#commits_by' do
|
||||
set(:project) { create(:project, :repository) }
|
||||
|
||||
shared_examples 'batch commits fetching' do
|
||||
let(:oids) { TestEnv::BRANCH_SHA.values }
|
||||
|
||||
subject { project.repository.commits_by(oids: oids) }
|
||||
|
||||
it 'finds each commit' do
|
||||
expect(subject).not_to include(nil)
|
||||
expect(subject.size).to eq(oids.size)
|
||||
end
|
||||
|
||||
it 'returns only Commit instances' do
|
||||
expect(subject).to all( be_a(Commit) )
|
||||
end
|
||||
|
||||
context 'when some commits are not found ' do
|
||||
let(:oids) do
|
||||
['deadbeef'] + TestEnv::BRANCH_SHA.values.first(10)
|
||||
end
|
||||
|
||||
it 'returns only found commits' do
|
||||
expect(subject).not_to include(nil)
|
||||
expect(subject.size).to eq(10)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when no oids are passed' do
|
||||
let(:oids) { [] }
|
||||
|
||||
it 'does not call #batch_by_oid' do
|
||||
expect(Gitlab::Git::Commit).not_to receive(:batch_by_oid)
|
||||
|
||||
subject
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when Gitaly list_commits_by_oid is enabled' do
|
||||
it_behaves_like 'batch commits fetching'
|
||||
end
|
||||
|
||||
context 'when Gitaly list_commits_by_oid is enabled', :disable_gitaly do
|
||||
it_behaves_like 'batch commits fetching'
|
||||
end
|
||||
end
|
||||
|
||||
describe '#find_commits_by_message' do
|
||||
shared_examples 'finding commits by message' do
|
||||
it 'returns commits with messages containing a given string' do
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe PipelineSerializer do
|
||||
set(:project) { create(:project, :repository) }
|
||||
set(:user) { create(:user) }
|
||||
|
||||
let(:serializer) do
|
||||
|
@ -16,7 +17,7 @@ describe PipelineSerializer do
|
|||
end
|
||||
|
||||
context 'when a single object is being serialized' do
|
||||
let(:resource) { create(:ci_empty_pipeline) }
|
||||
let(:resource) { create(:ci_empty_pipeline, project: project) }
|
||||
|
||||
it 'serializers the pipeline object' do
|
||||
expect(subject[:id]).to eq resource.id
|
||||
|
@ -24,7 +25,7 @@ describe PipelineSerializer do
|
|||
end
|
||||
|
||||
context 'when multiple objects are being serialized' do
|
||||
let(:resource) { create_list(:ci_pipeline, 2) }
|
||||
let(:resource) { create_list(:ci_pipeline, 2, project: project) }
|
||||
|
||||
it 'serializers the array of pipelines' do
|
||||
expect(subject).not_to be_empty
|
||||
|
@ -100,7 +101,6 @@ describe PipelineSerializer do
|
|||
|
||||
context 'number of queries' do
|
||||
let(:resource) { Ci::Pipeline.all }
|
||||
let(:project) { create(:project) }
|
||||
|
||||
before do
|
||||
# Since RequestStore.active? is true we have to allow the
|
||||
|
|
Loading…
Reference in New Issue