Merge branch '23096-expire-artifacts-per-job' into 'master'
ExpireBuildArtifactsWorker query builds table without ordering enqueuing one job… See merge request !6732
This commit is contained in:
commit
c901936a82
|
@ -7,6 +7,7 @@ v 8.13.0 (unreleased)
|
||||||
- Use gitlab-shell v3.6.2 (GIT TRACE logging)
|
- Use gitlab-shell v3.6.2 (GIT TRACE logging)
|
||||||
- Add `/projects/visible` API endpoint (Ben Boeckel)
|
- Add `/projects/visible` API endpoint (Ben Boeckel)
|
||||||
- Fix centering of custom header logos (Ashley Dumaine)
|
- Fix centering of custom header logos (Ashley Dumaine)
|
||||||
|
- ExpireBuildArtifactsWorker query builds table without ordering enqueuing one job per build to cleanup
|
||||||
- AbstractReferenceFilter caches project_refs on RequestStore when active
|
- AbstractReferenceFilter caches project_refs on RequestStore when active
|
||||||
- Replaced the check sign to arrow in the show build view. !6501
|
- Replaced the check sign to arrow in the show build view. !6501
|
||||||
- Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar)
|
- Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar)
|
||||||
|
|
|
@ -2,12 +2,11 @@ class ExpireBuildArtifactsWorker
|
||||||
include Sidekiq::Worker
|
include Sidekiq::Worker
|
||||||
|
|
||||||
def perform
|
def perform
|
||||||
Rails.logger.info 'Cleaning old build artifacts'
|
Rails.logger.info 'Scheduling removal of build artifacts'
|
||||||
|
|
||||||
builds = Ci::Build.with_expired_artifacts
|
build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
|
||||||
builds.find_each(batch_size: 50).each do |build|
|
build_ids = build_ids.map { |build_id| [build_id] }
|
||||||
Rails.logger.debug "Removing artifacts build #{build.id}..."
|
|
||||||
build.erase_artifacts!
|
Sidekiq::Client.push_bulk('class' => ExpireBuildInstanceArtifactsWorker, 'args' => build_ids )
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
class ExpireBuildInstanceArtifactsWorker
|
||||||
|
include Sidekiq::Worker
|
||||||
|
|
||||||
|
def perform(build_id)
|
||||||
|
build = Ci::Build.with_expired_artifacts.reorder(nil).find_by(id: build_id)
|
||||||
|
return unless build
|
||||||
|
|
||||||
|
Rails.logger.info "Removing artifacts build #{build.id}..."
|
||||||
|
build.erase_artifacts!
|
||||||
|
end
|
||||||
|
end
|
|
@ -5,65 +5,42 @@ describe ExpireBuildArtifactsWorker do
|
||||||
|
|
||||||
let(:worker) { described_class.new }
|
let(:worker) { described_class.new }
|
||||||
|
|
||||||
|
before { Sidekiq::Worker.clear_all }
|
||||||
|
|
||||||
describe '#perform' do
|
describe '#perform' do
|
||||||
before { build }
|
before { build }
|
||||||
|
|
||||||
subject! { worker.perform }
|
subject! do
|
||||||
|
Sidekiq::Testing.fake! { worker.perform }
|
||||||
|
end
|
||||||
|
|
||||||
context 'with expired artifacts' do
|
context 'with expired artifacts' do
|
||||||
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
|
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
|
||||||
|
|
||||||
it 'does expire' do
|
it 'enqueues that build' do
|
||||||
expect(build.reload.artifacts_expired?).to be_truthy
|
expect(jobs_enqueued.size).to eq(1)
|
||||||
end
|
expect(jobs_enqueued[0]["args"]).to eq([build.id])
|
||||||
|
|
||||||
it 'does remove files' do
|
|
||||||
expect(build.reload.artifacts_file.exists?).to be_falsey
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'does nullify artifacts_file column' do
|
|
||||||
expect(build.reload.artifacts_file_identifier).to be_nil
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'with not yet expired artifacts' do
|
context 'with not yet expired artifacts' do
|
||||||
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
|
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
|
||||||
|
|
||||||
it 'does not expire' do
|
it 'does not enqueue that build' do
|
||||||
expect(build.reload.artifacts_expired?).to be_falsey
|
expect(jobs_enqueued.size).to eq(0)
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not remove files' do
|
|
||||||
expect(build.reload.artifacts_file.exists?).to be_truthy
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not nullify artifacts_file column' do
|
|
||||||
expect(build.reload.artifacts_file_identifier).not_to be_nil
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'without expire date' do
|
context 'without expire date' do
|
||||||
let(:build) { create(:ci_build, :artifacts) }
|
let(:build) { create(:ci_build, :artifacts) }
|
||||||
|
|
||||||
it 'does not expire' do
|
it 'does not enqueue that build' do
|
||||||
expect(build.reload.artifacts_expired?).to be_falsey
|
expect(jobs_enqueued.size).to eq(0)
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not remove files' do
|
|
||||||
expect(build.reload.artifacts_file.exists?).to be_truthy
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'does not nullify artifacts_file column' do
|
|
||||||
expect(build.reload.artifacts_file_identifier).not_to be_nil
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'for expired artifacts' do
|
def jobs_enqueued
|
||||||
let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) }
|
Sidekiq::Queues.jobs_by_worker['ExpireBuildInstanceArtifactsWorker']
|
||||||
|
|
||||||
it 'is still expired' do
|
|
||||||
expect(build.reload.artifacts_expired?).to be_truthy
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,69 @@
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
describe ExpireBuildInstanceArtifactsWorker do
|
||||||
|
include RepoHelpers
|
||||||
|
|
||||||
|
let(:worker) { described_class.new }
|
||||||
|
|
||||||
|
describe '#perform' do
|
||||||
|
before { build }
|
||||||
|
|
||||||
|
subject! { worker.perform(build.id) }
|
||||||
|
|
||||||
|
context 'with expired artifacts' do
|
||||||
|
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
|
||||||
|
|
||||||
|
it 'does expire' do
|
||||||
|
expect(build.reload.artifacts_expired?).to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does remove files' do
|
||||||
|
expect(build.reload.artifacts_file.exists?).to be_falsey
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does nullify artifacts_file column' do
|
||||||
|
expect(build.reload.artifacts_file_identifier).to be_nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'with not yet expired artifacts' do
|
||||||
|
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
|
||||||
|
|
||||||
|
it 'does not expire' do
|
||||||
|
expect(build.reload.artifacts_expired?).to be_falsey
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not remove files' do
|
||||||
|
expect(build.reload.artifacts_file.exists?).to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not nullify artifacts_file column' do
|
||||||
|
expect(build.reload.artifacts_file_identifier).not_to be_nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'without expire date' do
|
||||||
|
let(:build) { create(:ci_build, :artifacts) }
|
||||||
|
|
||||||
|
it 'does not expire' do
|
||||||
|
expect(build.reload.artifacts_expired?).to be_falsey
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not remove files' do
|
||||||
|
expect(build.reload.artifacts_file.exists?).to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not nullify artifacts_file column' do
|
||||||
|
expect(build.reload.artifacts_file_identifier).not_to be_nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'for expired artifacts' do
|
||||||
|
let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) }
|
||||||
|
|
||||||
|
it 'is still expired' do
|
||||||
|
expect(build.reload.artifacts_expired?).to be_truthy
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue