From 382a1ef45360b538a80fd5d34cde71f53522dc2a Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 6 Dec 2017 15:11:38 +0100
Subject: [PATCH 1/3] Add invalid builds counter metric to stage seeds class

---
 lib/gitlab/ci/stage/seed.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb
index bc97aa63b02..6a3b2f6cc4a 100644
--- a/lib/gitlab/ci/stage/seed.rb
+++ b/lib/gitlab/ci/stage/seed.rb
@@ -39,6 +39,10 @@ module Gitlab
           pipeline.stages.create!(stage).tap do |stage|
             builds_attributes = builds.map do |attributes|
               attributes.merge(stage_id: stage.id)
+
+              if attributes.fetch(:stage_id).nil?
+                invalid_builds_counter.increment(node: hostname)
+              end
             end
 
             pipeline.builds.create!(builds_attributes).each do |build|
@@ -52,6 +56,15 @@ module Gitlab
         def protected_ref?
           @protected_ref ||= project.protected_for?(pipeline.ref)
         end
+
+        def invalid_builds_counter
+          @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter,
+                                               'Builds without stage assigned counter')
+        end
+
+        def hostname
+          @hostname ||= Socket.gethostname
+        end
       end
     end
   end

From 5ccced63122fe2d21fff7d7298db1081776193b4 Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Wed, 6 Dec 2017 15:30:20 +0100
Subject: [PATCH 2/3] Move invalid builds counter out of the transaction

---
 lib/gitlab/ci/pipeline/chain/create.rb | 17 +++++++++++++++++
 lib/gitlab/ci/stage/seed.rb            | 13 -------------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index d5e17a123df..42ae1650437 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -17,11 +17,28 @@ module Gitlab
             end
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
+          ensure
+            pipeline.builds.find_each do |build|
+              next if build.stage_id.present?
+
+              invalid_builds_counter.increment(node: hostname)
+            end
           end
 
           def break?
             !pipeline.persisted?
           end
+
+          private
+
+          def invalid_builds_counter
+            @counter ||= Gitlab::Metrics
+              .counter(:invalid_builds_counter, 'Invalid builds counter')
+          end
+
+          def hostname
+            @hostname ||= Socket.gethostname
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb
index 6a3b2f6cc4a..bc97aa63b02 100644
--- a/lib/gitlab/ci/stage/seed.rb
+++ b/lib/gitlab/ci/stage/seed.rb
@@ -39,10 +39,6 @@ module Gitlab
           pipeline.stages.create!(stage).tap do |stage|
             builds_attributes = builds.map do |attributes|
               attributes.merge(stage_id: stage.id)
-
-              if attributes.fetch(:stage_id).nil?
-                invalid_builds_counter.increment(node: hostname)
-              end
             end
 
             pipeline.builds.create!(builds_attributes).each do |build|
@@ -56,15 +52,6 @@ module Gitlab
         def protected_ref?
           @protected_ref ||= project.protected_for?(pipeline.ref)
         end
-
-        def invalid_builds_counter
-          @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter,
-                                               'Builds without stage assigned counter')
-        end
-
-        def hostname
-          @hostname ||= Socket.gethostname
-        end
       end
     end
   end

From be12f3ed24b7c2d120e249d52179eb09bff7c8aa Mon Sep 17 00:00:00 2001
From: Grzegorz Bizon <grzesiek.bizon@gmail.com>
Date: Thu, 7 Dec 2017 10:27:07 +0100
Subject: [PATCH 3/3] Update pipeline create chain Prometheus metric

---
 lib/gitlab/ci/pipeline/chain/create.rb | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 42ae1650437..d19a2519803 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -18,9 +18,7 @@ module Gitlab
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
           ensure
-            pipeline.builds.find_each do |build|
-              next if build.stage_id.present?
-
+            if pipeline.builds.where(stage_id: nil).any?
               invalid_builds_counter.increment(node: hostname)
             end
           end
@@ -33,7 +31,8 @@ module Gitlab
 
           def invalid_builds_counter
             @counter ||= Gitlab::Metrics
-              .counter(:invalid_builds_counter, 'Invalid builds counter')
+              .counter(:gitlab_ci_invalid_builds_total,
+                       'Invalid builds without stage assigned counter')
           end
 
           def hostname