From 0dd202007f902bbde38498f2f424bd4a5c06d814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 6 Feb 2018 23:01:20 +0100 Subject: [PATCH 1/8] Implement billing_enabled change counter --- .../check_gcp_project_billing_worker.rb | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/app/workers/check_gcp_project_billing_worker.rb b/app/workers/check_gcp_project_billing_worker.rb index 5466ccdda59..c12211e13ea 100644 --- a/app/workers/check_gcp_project_billing_worker.rb +++ b/app/workers/check_gcp_project_billing_worker.rb @@ -34,6 +34,9 @@ class CheckGcpProjectBillingWorker return unless try_obtain_lease_for(token) billing_enabled_projects = CheckGcpProjectBillingService.new.execute(token) + + update_billing_change_counter(check_previous_state(token), !billing_enabled_projects.empty?) + Gitlab::Redis::SharedState.with do |redis| redis.set(self.class.redis_shared_state_key_for(token), !billing_enabled_projects.empty?, @@ -51,9 +54,27 @@ class CheckGcpProjectBillingWorker "gitlab:gcp:session:#{token_key}" end + def self.redis_billing_change_key + "gitlab:gcp:billing_enabled_changes" + end + def try_obtain_lease_for(token) Gitlab::ExclusiveLease .new("check_gcp_project_billing_worker:#{token.hash}", timeout: LEASE_TIMEOUT) .try_obtain end + + def check_previous_state(token) + Gitlab::Redis::SharedState.with do |redis| + redis.get(self.class.redis_shared_state_key_for(token)) + end + end + + def update_billing_change_counter(previous_state, current_state) + return unless previous_state == 'false' && current_state + + Gitlab::Redis::SharedState.with do |redis| + redis.incr(self.class.redis_billing_change_key) + end + end end From 7f430a91bd839a396001f25fd8174f8b4d37aca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 6 Feb 2018 23:01:36 +0100 Subject: [PATCH 2/8] Add specs for billing_enabled change counter --- .../check_gcp_project_billing_worker_spec.rb | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/spec/workers/check_gcp_project_billing_worker_spec.rb b/spec/workers/check_gcp_project_billing_worker_spec.rb index 7b7a7c1bc44..5878f5cefa2 100644 --- a/spec/workers/check_gcp_project_billing_worker_spec.rb +++ b/spec/workers/check_gcp_project_billing_worker_spec.rb @@ -6,6 +6,11 @@ describe CheckGcpProjectBillingWorker do subject { described_class.new.perform('token_key') } + before do + allow_any_instance_of(described_class).to receive(:check_previous_state) + allow_any_instance_of(described_class).to receive(:update_billing_change_counter) + end + context 'when there is a token in redis' do before do allow(described_class).to receive(:get_session_token).and_return(token) @@ -58,4 +63,65 @@ describe CheckGcpProjectBillingWorker do end end end + + describe 'billing change counter' do + subject { described_class.new.perform('token_key') } + + before do + allow(described_class).to receive(:get_session_token).and_return('bogustoken') + allow_any_instance_of(described_class).to receive(:try_obtain_lease_for).and_return('randomuuid') + Gitlab::Redis::SharedState.with do |redis| + allow(redis).to receive(:set) + end + end + + context 'when previous state was false' do + before do + expect_any_instance_of(described_class).to receive(:check_previous_state).and_return('false') + end + + context 'when the current state is false' do + before do + expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([]) + end + + it 'does not increment the billing change counter' do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).not_to receive(:incr) + end + + subject + end + end + + context 'when the current state is true' do + before do + expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([double]) + end + + it 'increments the billing change counter' do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).to receive(:incr) + end + + subject + end + end + end + + context 'when previous state was true' do + before do + expect_any_instance_of(described_class).to receive(:check_previous_state).and_return('true') + expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([double]) + end + + it 'does not increment the billing change counter' do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).not_to receive(:incr) + end + + subject + end + end + end end From c00a17f653aaf1c0ee69d2726be182d015d3df5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 6 Feb 2018 23:22:47 +0100 Subject: [PATCH 3/8] Add missing newline in CheckGcpProjectBillingWorker spec --- spec/workers/check_gcp_project_billing_worker_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/workers/check_gcp_project_billing_worker_spec.rb b/spec/workers/check_gcp_project_billing_worker_spec.rb index 5878f5cefa2..5f473d05e60 100644 --- a/spec/workers/check_gcp_project_billing_worker_spec.rb +++ b/spec/workers/check_gcp_project_billing_worker_spec.rb @@ -70,6 +70,7 @@ describe CheckGcpProjectBillingWorker do before do allow(described_class).to receive(:get_session_token).and_return('bogustoken') allow_any_instance_of(described_class).to receive(:try_obtain_lease_for).and_return('randomuuid') + Gitlab::Redis::SharedState.with do |redis| allow(redis).to receive(:set) end From bafab35e84e88f77d850ea13973d8fddc110225a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 7 Feb 2018 17:41:03 +0100 Subject: [PATCH 4/8] Use Prometheus counter instead of redis --- app/workers/check_gcp_project_billing_worker.rb | 11 ++++++----- .../workers/check_gcp_project_billing_worker_spec.rb | 12 +++--------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/app/workers/check_gcp_project_billing_worker.rb b/app/workers/check_gcp_project_billing_worker.rb index c12211e13ea..c3a7929bd14 100644 --- a/app/workers/check_gcp_project_billing_worker.rb +++ b/app/workers/check_gcp_project_billing_worker.rb @@ -54,8 +54,11 @@ class CheckGcpProjectBillingWorker "gitlab:gcp:session:#{token_key}" end - def self.redis_billing_change_key - "gitlab:gcp:billing_enabled_changes" + def billing_changed_counter + @billing_changed_counter ||= Gitlab::Metrics.counter( + :gcp_billing_change_count, + "Counts the number of times a GCP project changed billing_enabled state from false to true" + ) end def try_obtain_lease_for(token) @@ -73,8 +76,6 @@ class CheckGcpProjectBillingWorker def update_billing_change_counter(previous_state, current_state) return unless previous_state == 'false' && current_state - Gitlab::Redis::SharedState.with do |redis| - redis.incr(self.class.redis_billing_change_key) - end + billing_changed_counter.increment end end diff --git a/spec/workers/check_gcp_project_billing_worker_spec.rb b/spec/workers/check_gcp_project_billing_worker_spec.rb index 5f473d05e60..132f9751f92 100644 --- a/spec/workers/check_gcp_project_billing_worker_spec.rb +++ b/spec/workers/check_gcp_project_billing_worker_spec.rb @@ -87,9 +87,7 @@ describe CheckGcpProjectBillingWorker do end it 'does not increment the billing change counter' do - Gitlab::Redis::SharedState.with do |redis| - expect(redis).not_to receive(:incr) - end + expect_any_instance_of(described_class).not_to receive(:billing_changed_counter) subject end @@ -101,9 +99,7 @@ describe CheckGcpProjectBillingWorker do end it 'increments the billing change counter' do - Gitlab::Redis::SharedState.with do |redis| - expect(redis).to receive(:incr) - end + expect_any_instance_of(described_class).to receive_message_chain(:billing_changed_counter, :increment) subject end @@ -117,9 +113,7 @@ describe CheckGcpProjectBillingWorker do end it 'does not increment the billing change counter' do - Gitlab::Redis::SharedState.with do |redis| - expect(redis).not_to receive(:incr) - end + expect_any_instance_of(described_class).not_to receive(:billing_changed_counter) subject end From 0215435058ea9f9ebcc1b793425fccd22317e651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 8 Feb 2018 16:57:14 +0100 Subject: [PATCH 5/8] Refactor CheckGcpProjectBillingWorker --- .../projects/clusters/gcp_controller.rb | 14 +++--- .../check_gcp_project_billing_worker.rb | 43 ++++++++++--------- .../check_gcp_project_billing_worker_spec.rb | 18 +++----- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/app/controllers/projects/clusters/gcp_controller.rb b/app/controllers/projects/clusters/gcp_controller.rb index 94d33b91562..0f41af7d87b 100644 --- a/app/controllers/projects/clusters/gcp_controller.rb +++ b/app/controllers/projects/clusters/gcp_controller.rb @@ -39,12 +39,12 @@ class Projects::Clusters::GcpController < Projects::ApplicationController def verify_billing case google_project_billing_status - when 'true' - return - when 'false' - flash[:alert] = _('Please enable billing for one of your projects to be able to create a Kubernetes cluster, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } - else + when nil flash[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.') + when false + flash[:alert] = _('Please enable billing for one of your projects to be able to create a Kubernetes cluster, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } + when true + return end @cluster = ::Clusters::Cluster.new(create_params) @@ -81,9 +81,7 @@ class Projects::Clusters::GcpController < Projects::ApplicationController end def google_project_billing_status - Gitlab::Redis::SharedState.with do |redis| - redis.get(CheckGcpProjectBillingWorker.redis_shared_state_key_for(token_in_session)) - end + CheckGcpProjectBillingWorker.get_billing_state(token_in_session) end def token_in_session diff --git a/app/workers/check_gcp_project_billing_worker.rb b/app/workers/check_gcp_project_billing_worker.rb index c3a7929bd14..1a2894f6268 100644 --- a/app/workers/check_gcp_project_billing_worker.rb +++ b/app/workers/check_gcp_project_billing_worker.rb @@ -22,8 +22,11 @@ class CheckGcpProjectBillingWorker end end - def self.redis_shared_state_key_for(token) - "gitlab:gcp:#{Digest::SHA1.hexdigest(token)}:billing_enabled" + def self.get_billing_state(token) + Gitlab::Redis::SharedState.with do |redis| + value = redis.get(redis_shared_state_key_for(token)) + ActiveRecord::Type::Boolean.new.type_cast_from_user(value) + end end def perform(token_key) @@ -33,15 +36,9 @@ class CheckGcpProjectBillingWorker return unless token return unless try_obtain_lease_for(token) - billing_enabled_projects = CheckGcpProjectBillingService.new.execute(token) - - update_billing_change_counter(check_previous_state(token), !billing_enabled_projects.empty?) - - Gitlab::Redis::SharedState.with do |redis| - redis.set(self.class.redis_shared_state_key_for(token), - !billing_enabled_projects.empty?, - ex: BILLING_TIMEOUT) - end + billing_enabled_state = !CheckGcpProjectBillingService.new.execute(token).empty? + update_billing_change_counter(self.class.get_billing_state(token), billing_enabled_state) + self.class.set_billing_state(token, billing_enabled_state) end private @@ -54,11 +51,14 @@ class CheckGcpProjectBillingWorker "gitlab:gcp:session:#{token_key}" end - def billing_changed_counter - @billing_changed_counter ||= Gitlab::Metrics.counter( - :gcp_billing_change_count, - "Counts the number of times a GCP project changed billing_enabled state from false to true" - ) + def self.redis_shared_state_key_for(token) + "gitlab:gcp:#{Digest::SHA1.hexdigest(token)}:billing_enabled" + end + + def self.set_billing_state(token, value) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key_for(token), value, ex: BILLING_TIMEOUT) + end end def try_obtain_lease_for(token) @@ -67,14 +67,15 @@ class CheckGcpProjectBillingWorker .try_obtain end - def check_previous_state(token) - Gitlab::Redis::SharedState.with do |redis| - redis.get(self.class.redis_shared_state_key_for(token)) - end + def billing_changed_counter + @billing_changed_counter ||= Gitlab::Metrics.counter( + :gcp_billing_change_count, + "Counts the number of times a GCP project changed billing_enabled state from false to true" + ) end def update_billing_change_counter(previous_state, current_state) - return unless previous_state == 'false' && current_state + return unless !previous_state && current_state billing_changed_counter.increment end diff --git a/spec/workers/check_gcp_project_billing_worker_spec.rb b/spec/workers/check_gcp_project_billing_worker_spec.rb index 132f9751f92..179f26475e1 100644 --- a/spec/workers/check_gcp_project_billing_worker_spec.rb +++ b/spec/workers/check_gcp_project_billing_worker_spec.rb @@ -7,7 +7,7 @@ describe CheckGcpProjectBillingWorker do subject { described_class.new.perform('token_key') } before do - allow_any_instance_of(described_class).to receive(:check_previous_state) + allow(described_class).to receive(:get_billing_state) allow_any_instance_of(described_class).to receive(:update_billing_change_counter) end @@ -28,11 +28,8 @@ describe CheckGcpProjectBillingWorker do end it 'stores billing status in redis' do - redis_double = double - expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([double]) - expect(Gitlab::Redis::SharedState).to receive(:with).and_yield(redis_double) - expect(redis_double).to receive(:set).with(described_class.redis_shared_state_key_for(token), anything, anything) + expect(described_class).to receive(:set_billing_state).with(token, true) subject end @@ -53,7 +50,7 @@ describe CheckGcpProjectBillingWorker do context 'when there is no token in redis' do before do - allow_any_instance_of(described_class).to receive(:get_session_token).and_return(nil) + allow(described_class).to receive(:get_session_token).and_return(nil) end it 'does not call the service' do @@ -70,15 +67,12 @@ describe CheckGcpProjectBillingWorker do before do allow(described_class).to receive(:get_session_token).and_return('bogustoken') allow_any_instance_of(described_class).to receive(:try_obtain_lease_for).and_return('randomuuid') - - Gitlab::Redis::SharedState.with do |redis| - allow(redis).to receive(:set) - end + allow(described_class).to receive(:set_billing_state) end context 'when previous state was false' do before do - expect_any_instance_of(described_class).to receive(:check_previous_state).and_return('false') + expect(described_class).to receive(:get_billing_state).and_return(false) end context 'when the current state is false' do @@ -108,7 +102,7 @@ describe CheckGcpProjectBillingWorker do context 'when previous state was true' do before do - expect_any_instance_of(described_class).to receive(:check_previous_state).and_return('true') + expect(described_class).to receive(:get_billing_state).and_return(true) expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([double]) end From 808ad1231c590efed642809cba6d155d8767d69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 8 Feb 2018 17:21:34 +0100 Subject: [PATCH 6/8] Log billing state changes in CheckGcpProjectBillingWorker --- app/workers/check_gcp_project_billing_worker.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/workers/check_gcp_project_billing_worker.rb b/app/workers/check_gcp_project_billing_worker.rb index 1a2894f6268..624d515c6bc 100644 --- a/app/workers/check_gcp_project_billing_worker.rb +++ b/app/workers/check_gcp_project_billing_worker.rb @@ -74,7 +74,20 @@ class CheckGcpProjectBillingWorker ) end + def log_transition(previous_state, current_state) + state_message = if previous_state.nil? && !current_state + "no_billing" + elsif previous_state.nil? && current_state + "with_billing" + elsif !previous_state && current_state + "billing_configured" + end + + Rails.logger.info "#{self.class}: state: #{state_message}" + end + def update_billing_change_counter(previous_state, current_state) + log_transition(previous_state, current_state) return unless !previous_state && current_state billing_changed_counter.increment From 5553524516135f573b1264785ef04f0fe638d5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 8 Feb 2018 18:04:52 +0100 Subject: [PATCH 7/8] Fix GCP cluster feature spec --- spec/features/projects/clusters/gcp_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/features/projects/clusters/gcp_spec.rb b/spec/features/projects/clusters/gcp_spec.rb index 02dbd3380b3..4d47cdb500c 100644 --- a/spec/features/projects/clusters/gcp_spec.rb +++ b/spec/features/projects/clusters/gcp_spec.rb @@ -25,7 +25,7 @@ feature 'Gcp Cluster', :js do context 'when user has a GCP project with billing enabled' do before do allow_any_instance_of(Projects::Clusters::GcpController).to receive(:authorize_google_project_billing) - allow_any_instance_of(Projects::Clusters::GcpController).to receive(:google_project_billing_status).and_return('true') + allow_any_instance_of(Projects::Clusters::GcpController).to receive(:google_project_billing_status).and_return(true) end context 'when user does not have a cluster and visits cluster index page' do @@ -134,7 +134,7 @@ feature 'Gcp Cluster', :js do context 'when user does not have a GCP project with billing enabled' do before do allow_any_instance_of(Projects::Clusters::GcpController).to receive(:authorize_google_project_billing) - allow_any_instance_of(Projects::Clusters::GcpController).to receive(:google_project_billing_status).and_return('false') + allow_any_instance_of(Projects::Clusters::GcpController).to receive(:google_project_billing_status).and_return(false) visit project_clusters_path(project) From f95d9fdcc5f52e6371ca78b21538e87e9ba738f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 12 Feb 2018 18:32:10 +0100 Subject: [PATCH 8/8] Count all billing_state transitions with labels --- .../check_gcp_project_billing_worker.rb | 27 +++++++++---------- .../check_gcp_project_billing_worker_spec.rb | 8 +++--- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/app/workers/check_gcp_project_billing_worker.rb b/app/workers/check_gcp_project_billing_worker.rb index 624d515c6bc..363f81590ab 100644 --- a/app/workers/check_gcp_project_billing_worker.rb +++ b/app/workers/check_gcp_project_billing_worker.rb @@ -7,6 +7,7 @@ class CheckGcpProjectBillingWorker LEASE_TIMEOUT = 3.seconds.to_i SESSION_KEY_TIMEOUT = 5.minutes BILLING_TIMEOUT = 1.hour + BILLING_CHANGED_LABELS = { state_transition: nil }.freeze def self.get_session_token(token_key) Gitlab::Redis::SharedState.with do |redis| @@ -70,26 +71,22 @@ class CheckGcpProjectBillingWorker def billing_changed_counter @billing_changed_counter ||= Gitlab::Metrics.counter( :gcp_billing_change_count, - "Counts the number of times a GCP project changed billing_enabled state from false to true" + "Counts the number of times a GCP project changed billing_enabled state from false to true", + BILLING_CHANGED_LABELS ) end - def log_transition(previous_state, current_state) - state_message = if previous_state.nil? && !current_state - "no_billing" - elsif previous_state.nil? && current_state - "with_billing" - elsif !previous_state && current_state - "billing_configured" - end - - Rails.logger.info "#{self.class}: state: #{state_message}" + def state_transition(previous_state, current_state) + if previous_state.nil? && !current_state + 'no_billing' + elsif previous_state.nil? && current_state + 'with_billing' + elsif !previous_state && current_state + 'billing_configured' + end end def update_billing_change_counter(previous_state, current_state) - log_transition(previous_state, current_state) - return unless !previous_state && current_state - - billing_changed_counter.increment + billing_changed_counter.increment(state_transition: state_transition(previous_state, current_state)) end end diff --git a/spec/workers/check_gcp_project_billing_worker_spec.rb b/spec/workers/check_gcp_project_billing_worker_spec.rb index 179f26475e1..526ecf75921 100644 --- a/spec/workers/check_gcp_project_billing_worker_spec.rb +++ b/spec/workers/check_gcp_project_billing_worker_spec.rb @@ -80,8 +80,8 @@ describe CheckGcpProjectBillingWorker do expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([]) end - it 'does not increment the billing change counter' do - expect_any_instance_of(described_class).not_to receive(:billing_changed_counter) + it 'increments the billing change counter' do + expect_any_instance_of(described_class).to receive_message_chain(:billing_changed_counter, :increment) subject end @@ -106,8 +106,8 @@ describe CheckGcpProjectBillingWorker do expect(CheckGcpProjectBillingService).to receive_message_chain(:new, :execute).and_return([double]) end - it 'does not increment the billing change counter' do - expect_any_instance_of(described_class).not_to receive(:billing_changed_counter) + it 'increment the billing change counter' do + expect_any_instance_of(described_class).to receive_message_chain(:billing_changed_counter, :increment) subject end