Merge branch 'increase-event-retention-prune-old-events-worker' into 'master'

Prune events older than 2 years instead of 12 months

See merge request gitlab-org/gitlab-ce!22145
This commit is contained in:
Douwe Maan 2018-10-08 10:41:09 +00:00
commit 651039ac5b
4 changed files with 22 additions and 12 deletions

View file

@ -6,14 +6,13 @@ class PruneOldEventsWorker
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def perform def perform
# Contribution calendar shows maximum 12 months of events. # Contribution calendar shows maximum 12 months of events, we retain 2 years for data integrity.
# Double nested query is used because MySQL doesn't allow DELETE subqueries # Double nested query is used because MySQL doesn't allow DELETE subqueries on the same table.
# on the same table.
Event.unscoped.where( Event.unscoped.where(
'(id IN (SELECT id FROM (?) ids_to_remove))', '(id IN (SELECT id FROM (?) ids_to_remove))',
Event.unscoped.where( Event.unscoped.where(
'created_at < ?', 'created_at < ?',
(12.months + 1.day).ago) (2.years + 1.day).ago)
.select(:id) .select(:id)
.limit(10_000)) .limit(10_000))
.delete_all .delete_all

View file

@ -0,0 +1,5 @@
---
title: Increased retained event data by extending events pruner timeframe to 2 years
merge_request: 22145
author:
type: changed

View file

@ -44,7 +44,7 @@ YYYY-MM-DD
### Event Time Period Limit ### Event Time Period Limit
GitLab removes events older than 1 year from the events table for performance reasons. The range of 1 year was chosen because user contribution calendars only show contributions of the past year. GitLab removes events older than 2 years from the events table for performance reasons.
## List currently authenticated user's events ## List currently authenticated user's events

View file

@ -4,23 +4,29 @@ describe PruneOldEventsWorker do
describe '#perform' do describe '#perform' do
let(:user) { create(:user) } let(:user) { create(:user) }
let!(:expired_event) { create(:event, :closed, author: user, created_at: 13.months.ago) } let!(:expired_event) { create(:event, :closed, author: user, created_at: 25.months.ago) }
let!(:not_expired_event) { create(:event, :closed, author: user, created_at: 1.day.ago) } let!(:not_expired_1_day_event) { create(:event, :closed, author: user, created_at: 1.day.ago) }
let!(:exactly_12_months_event) { create(:event, :closed, author: user, created_at: 12.months.ago) } let!(:not_expired_13_month_event) { create(:event, :closed, author: user, created_at: 13.months.ago) }
let!(:not_expired_2_years_event) { create(:event, :closed, author: user, created_at: 2.years.ago) }
it 'prunes events older than 12 months' do it 'prunes events older than 2 years' do
expect { subject.perform }.to change { Event.count }.by(-1) expect { subject.perform }.to change { Event.count }.by(-1)
expect(Event.find_by(id: expired_event.id)).to be_nil expect(Event.find_by(id: expired_event.id)).to be_nil
end end
it 'leaves fresh events' do it 'leaves fresh events' do
subject.perform subject.perform
expect(not_expired_event.reload).to be_present expect(not_expired_1_day_event.reload).to be_present
end end
it 'leaves events from exactly 12 months ago' do it 'leaves events from 13 months ago' do
subject.perform subject.perform
expect(exactly_12_months_event).to be_present expect(not_expired_13_month_event.reload).to be_present
end
it 'leaves events from 2 years ago' do
subject.perform
expect(not_expired_2_years_event).to be_present
end end
end end
end end