Optimise DAG processing
This commit is contained in:
parent
a55869483d
commit
cee2f86d57
|
@ -40,8 +40,11 @@ class CommitStatus < ApplicationRecord
|
||||||
scope :ordered, -> { order(:name) }
|
scope :ordered, -> { order(:name) }
|
||||||
scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) }
|
scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) }
|
||||||
scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
|
scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
|
||||||
|
scope :before_stage, -> (index) { where('stage_idx < ?', index) }
|
||||||
|
scope :for_stage, -> (index) { where(stage_idx: index) }
|
||||||
scope :after_stage, -> (index) { where('stage_idx > ?', index) }
|
scope :after_stage, -> (index) { where('stage_idx > ?', index) }
|
||||||
scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) }
|
scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) }
|
||||||
|
scope :for_ids, -> (ids) { where(id: ids) }
|
||||||
|
|
||||||
scope :with_needs, -> (names = nil) do
|
scope :with_needs, -> (names = nil) do
|
||||||
needs = Ci::BuildNeed.scoped_build.select(1)
|
needs = Ci::BuildNeed.scoped_build.select(1)
|
||||||
|
@ -49,8 +52,10 @@ class CommitStatus < ApplicationRecord
|
||||||
where('EXISTS (?)', needs).preload(:needs)
|
where('EXISTS (?)', needs).preload(:needs)
|
||||||
end
|
end
|
||||||
|
|
||||||
scope :without_needs, -> do
|
scope :without_needs, -> (names = nil) do
|
||||||
where('NOT EXISTS (?)', Ci::BuildNeed.scoped_build.select(1))
|
needs = Ci::BuildNeed.scoped_build.select(1)
|
||||||
|
needs = needs.where(name: names) if names
|
||||||
|
where('NOT EXISTS (?)', needs)
|
||||||
end
|
end
|
||||||
|
|
||||||
# We use `CommitStatusEnums.failure_reasons` here so that EE can more easily
|
# We use `CommitStatusEnums.failure_reasons` here so that EE can more easily
|
||||||
|
@ -149,6 +154,18 @@ class CommitStatus < ApplicationRecord
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def self.names
|
||||||
|
select(:name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.status_for_prior_stages(index)
|
||||||
|
before_stage(index).latest.status || 'success'
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.status_for_names(names)
|
||||||
|
where(name: names).latest.status || 'success'
|
||||||
|
end
|
||||||
|
|
||||||
def locking_enabled?
|
def locking_enabled?
|
||||||
will_save_change_to_status?
|
will_save_change_to_status?
|
||||||
end
|
end
|
||||||
|
|
|
@ -106,10 +106,15 @@ module HasStatus
|
||||||
scope :running_or_pending, -> { with_status(:running, :pending) }
|
scope :running_or_pending, -> { with_status(:running, :pending) }
|
||||||
scope :finished, -> { with_status(:success, :failed, :canceled) }
|
scope :finished, -> { with_status(:success, :failed, :canceled) }
|
||||||
scope :failed_or_canceled, -> { with_status(:failed, :canceled) }
|
scope :failed_or_canceled, -> { with_status(:failed, :canceled) }
|
||||||
|
scope :incomplete, -> { without_statuses(completed_statuses) }
|
||||||
|
|
||||||
scope :cancelable, -> do
|
scope :cancelable, -> do
|
||||||
where(status: [:running, :preparing, :pending, :created, :scheduled])
|
where(status: [:running, :preparing, :pending, :created, :scheduled])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
scope :without_statuses, -> (names) do
|
||||||
|
with_status(all_state_names - names.to_a)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def started?
|
def started?
|
||||||
|
|
|
@ -42,14 +42,19 @@ module Ci
|
||||||
return false unless trigger_build_ids.present?
|
return false unless trigger_build_ids.present?
|
||||||
return false unless Feature.enabled?(:ci_dag_support, project)
|
return false unless Feature.enabled?(:ci_dag_support, project)
|
||||||
|
|
||||||
# rubocop: disable CodeReuse/ActiveRecord
|
# we find processables that are dependent:
|
||||||
trigger_build_names = pipeline.statuses
|
# 1. because of current dependency,
|
||||||
.where(id: trigger_build_ids)
|
trigger_build_names = pipeline.processables.latest
|
||||||
.select(:name)
|
.for_ids(trigger_build_ids).names
|
||||||
# rubocop: enable CodeReuse/ActiveRecord
|
|
||||||
|
|
||||||
|
# 2. does not have builds that not yet complete
|
||||||
|
incomplete_build_names = pipeline.processables.latest
|
||||||
|
.incomplete.names
|
||||||
|
|
||||||
|
# Each found processable is guaranteed here to have completed status
|
||||||
created_processables
|
created_processables
|
||||||
.with_needs(trigger_build_names)
|
.with_needs(trigger_build_names)
|
||||||
|
.without_needs(incomplete_build_names)
|
||||||
.find_each
|
.find_each
|
||||||
.map(&method(:process_build_with_needs))
|
.map(&method(:process_build_with_needs))
|
||||||
.any?
|
.any?
|
||||||
|
@ -70,17 +75,13 @@ module Ci
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# rubocop: disable CodeReuse/ActiveRecord
|
|
||||||
def status_for_prior_stages(index)
|
def status_for_prior_stages(index)
|
||||||
pipeline.builds.where('stage_idx < ?', index).latest.status || 'success'
|
pipeline.processables.status_for_prior_stages(index)
|
||||||
end
|
end
|
||||||
# rubocop: enable CodeReuse/ActiveRecord
|
|
||||||
|
|
||||||
# rubocop: disable CodeReuse/ActiveRecord
|
|
||||||
def status_for_build_needs(needs)
|
def status_for_build_needs(needs)
|
||||||
pipeline.builds.where(name: needs).latest.status || 'success'
|
pipeline.processables.status_for_names(needs)
|
||||||
end
|
end
|
||||||
# rubocop: enable CodeReuse/ActiveRecord
|
|
||||||
|
|
||||||
# rubocop: disable CodeReuse/ActiveRecord
|
# rubocop: disable CodeReuse/ActiveRecord
|
||||||
def stage_indexes_of_created_processables_without_needs
|
def stage_indexes_of_created_processables_without_needs
|
||||||
|
@ -89,12 +90,10 @@ module Ci
|
||||||
end
|
end
|
||||||
# rubocop: enable CodeReuse/ActiveRecord
|
# rubocop: enable CodeReuse/ActiveRecord
|
||||||
|
|
||||||
# rubocop: disable CodeReuse/ActiveRecord
|
|
||||||
def created_processables_in_stage_without_needs(index)
|
def created_processables_in_stage_without_needs(index)
|
||||||
created_processables_without_needs
|
created_processables_without_needs
|
||||||
.where(stage_idx: index)
|
.for_stage(index)
|
||||||
end
|
end
|
||||||
# rubocop: enable CodeReuse/ActiveRecord
|
|
||||||
|
|
||||||
def created_processables_without_needs
|
def created_processables_without_needs
|
||||||
if Feature.enabled?(:ci_dag_support, project)
|
if Feature.enabled?(:ci_dag_support, project)
|
||||||
|
|
Loading…
Reference in New Issue