From 812dd06d512ab7774b375ce45aa9235aafc99911 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 4 May 2018 17:02:08 +0900 Subject: [PATCH] Introduce Redis helpers. Rename BuildTraceChunkFlushToDbWorker to Ci::BuildTraceChunkFlushWorker. --- app/models/ci/build_trace_chunk.rb | 2 +- app/workers/all_queues.yml | 2 +- .../build_trace_chunk_flush_to_db_worker.rb | 12 ------------ .../ci/build_trace_chunk_flush_worker.rb | 13 +++++++++++++ spec/models/ci/build_trace_chunk_spec.rb | 4 ++-- spec/requests/api/runner_spec.rb | 6 +++--- spec/spec_helper.rb | 19 +++++++++++++------ spec/support/chunked_io/chunked_io_helpers.rb | 5 ----- spec/support/redis/redis_helpers.rb | 18 ++++++++++++++++++ 9 files changed, 51 insertions(+), 30 deletions(-) delete mode 100644 app/workers/build_trace_chunk_flush_to_db_worker.rb create mode 100644 app/workers/ci/build_trace_chunk_flush_worker.rb create mode 100644 spec/support/redis/redis_helpers.rb diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index ee999a3395d..870b4ae2033 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -99,7 +99,7 @@ module Ci def schedule_to_db return if db? - BuildTraceChunkFlushToDbWorker.perform_async(id) + Ci::BuildTraceChunkFlushWorker.perform_async(id) end def fullfilled? diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 0d30971dfd2..dc628bb7f50 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -66,7 +66,7 @@ - pipeline_processing:pipeline_update - pipeline_processing:stage_update - pipeline_processing:update_head_pipeline_for_merge_request -- pipeline_processing:build_trace_chunk_flush_to_db +- pipeline_processing:ci_build_trace_chunk_flush - repository_check:repository_check_clear - repository_check:repository_check_single_repository diff --git a/app/workers/build_trace_chunk_flush_to_db_worker.rb b/app/workers/build_trace_chunk_flush_to_db_worker.rb deleted file mode 100644 index 495118f7c14..00000000000 --- a/app/workers/build_trace_chunk_flush_to_db_worker.rb +++ /dev/null @@ -1,12 +0,0 @@ -class BuildTraceChunkFlushToDbWorker - include ApplicationWorker - include PipelineQueue - - queue_namespace :pipeline_processing - - def perform(build_trace_chunk_id) - Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk| - build_trace_chunk.use_database! - end - end -end diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb new file mode 100644 index 00000000000..b8f8be29c7b --- /dev/null +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -0,0 +1,13 @@ +module Ci + class BuildTraceChunkFlushWorker + include ApplicationWorker + + queue_namespace :pipeline_processing + + def perform(build_trace_chunk_id) + ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk| + build_trace_chunk.use_database! + end + end + end +end diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index 991f501f633..118b72da11c 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -75,7 +75,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do let(:value) { 'a' * described_class::CHUNK_SIZE } it 'schedules stashing data' do - expect(BuildTraceChunkFlushToDbWorker).to receive(:perform_async).once + expect(Ci::BuildTraceChunkFlushWorker).to receive(:perform_async).once subject end @@ -112,7 +112,7 @@ describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do context 'when fullfilled chunk size' do it 'does not schedule stashing data' do - expect(BuildTraceChunkFlushToDbWorker).not_to receive(:perform_async) + expect(Ci::BuildTraceChunkFlushWorker).not_to receive(:perform_async) subject end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index f27c95b4907..ff23b625244 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe API::Runner, :clean_gitlab_redis_shared_state do include StubGitlabCalls - include ChunkedIOHelpers + include RedisHelpers let(:registration_token) { 'abcdefg123456' } @@ -873,8 +873,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do patch_the_trace expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended' - # GitLab-Rails enxounters an outage on Redis - redis_shared_state_outage! + # GitLab-Rails encounters an outage on Redis + redis_shared_state_cleanup! expect(job.reload.trace.raw).to eq '' # GitLab-Runner patchs diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cc61cd7d838..b4fc596a751 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -86,6 +86,7 @@ RSpec.configure do |config| config.include WaitForRequests, :js config.include LiveDebugger, :js config.include MigrationsHelpers, :migration + config.include RedisHelpers if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. @@ -146,21 +147,27 @@ RSpec.configure do |config| end config.around(:each, :clean_gitlab_redis_cache) do |example| - Gitlab::Redis::Cache.with(&:flushall) + redis_cache_cleanup! example.run - Gitlab::Redis::Cache.with(&:flushall) + redis_cache_cleanup! end config.around(:each, :clean_gitlab_redis_shared_state) do |example| - Gitlab::Redis::SharedState.with(&:flushall) - Sidekiq.redis(&:flushall) + redis_shared_state_cleanup! example.run - Gitlab::Redis::SharedState.with(&:flushall) - Sidekiq.redis(&:flushall) + redis_shared_state_cleanup! + end + + config.around(:each, :clean_gitlab_redis_queues) do |example| + redis_queues_cleanup! + + example.run + + redis_queues_cleanup! end # The :each scope runs "inside" the example, so this hook ensures the DB is in the diff --git a/spec/support/chunked_io/chunked_io_helpers.rb b/spec/support/chunked_io/chunked_io_helpers.rb index 4238a4b3e94..fec1f951563 100644 --- a/spec/support/chunked_io/chunked_io_helpers.rb +++ b/spec/support/chunked_io/chunked_io_helpers.rb @@ -8,9 +8,4 @@ module ChunkedIOHelpers stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size) stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size) end - - def redis_shared_state_outage! - Gitlab::Redis::SharedState.with(&:flushall) - Sidekiq.redis(&:flushall) - end end diff --git a/spec/support/redis/redis_helpers.rb b/spec/support/redis/redis_helpers.rb new file mode 100644 index 00000000000..0457e8487d8 --- /dev/null +++ b/spec/support/redis/redis_helpers.rb @@ -0,0 +1,18 @@ +module RedisHelpers + # config/README.md + + # Usage: performance enhancement + def redis_cache_cleanup! + Gitlab::Redis::Cache.with(&:flushall) + end + + # Usage: SideKiq, Mailroom, CI Runner, Workhorse, push services + def redis_queues_cleanup! + Gitlab::Redis::Queues.with(&:flushall) + end + + # Usage: session state, rate limiting + def redis_shared_state_cleanup! + Gitlab::Redis::SharedState.with(&:flushall) + end +end