Add a post-deploy migration to migrate from former Redis activity to DB

Signed-off-by: Rémy Coutable <remy@rymai.me>

db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb

@@ -0,0 +1,87 @@
+class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  USER_ACTIVITY_SET_KEY = 'user/activities'.freeze
+  ACTIVITIES_PER_PAGE = 100
+  TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED = Time.utc(2016, 12, 1)
+
+  def up
+    return if activities_count(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).zero?
+
+    day = Time.at(activities(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).first.second)
+
+    transaction do
+      while day <= Time.now.utc.tomorrow
+        persist_last_activity_on(day: day)
+        day = day.tomorrow
+      end
+    end
+  end
+
+  def down
+    # This ensures we don't lock all users for the duration of the migration.
+    update_column_in_batches(:users, :last_activity_on, nil) do |table, query|
+      query.where(table[:last_activity_on].not_eq(nil))
+    end
+  end
+
+  private
+
+  def persist_last_activity_on(day:, page: 1)
+    activities_count = activities_count(day.at_beginning_of_day, day.at_end_of_day)
+
+    return if activities_count.zero?
+
+    activities = activities(day.at_beginning_of_day, day.at_end_of_day, page: page)
+
+    update_sql =
+      Arel::UpdateManager.new(ActiveRecord::Base).
+        table(users_table).
+        set(users_table[:last_activity_on] => day.to_date).
+        where(users_table[:username].in(activities.map(&:first))).
+        to_sql
+
+    connection.exec_update(update_sql, self.class.name, [])
+
+    unless last_page?(page, activities_count)
+      persist_last_activity_on(day: day, page: page + 1)
+    end
+  end
+
+  def users_table
+    @users_table ||= Arel::Table.new(:users)
+  end
+
+  def activities(from, to, page: 1)
+    Gitlab::Redis.with do |redis|
+      redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
+        with_scores: true,
+        limit: limit(page))
+    end
+  end
+
+  def activities_count(from, to)
+    Gitlab::Redis.with do |redis|
+      redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
+    end
+  end
+
+  def limit(page)
+    [offset(page), ACTIVITIES_PER_PAGE]
+  end
+
+  def total_pages(count)
+    (count.to_f / ACTIVITIES_PER_PAGE).ceil
+  end
+
+  def last_page?(page, count)
+    page >= total_pages(count)
+  end
+
+  def offset(page)
+    (page - 1) * ACTIVITIES_PER_PAGE
+  end
+end

spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
+
+describe MigrateUserActivitiesToUsersLastActivityOn, :redis do
+  let(:migration) { described_class.new }
+  let!(:user_active_1) { create(:user) }
+  let!(:user_active_2) { create(:user) }
+
+  def record_activity(user, time)
+    Gitlab::Redis.with do |redis|
+      redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
+    end
+  end
+
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  before do
+    record_activity(user_active_1, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months)
+    record_activity(user_active_2, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months)
+    mute_stdout { migration.up }
+  end
+
+  describe '#up' do
+    it 'fills last_activity_on from the legacy Redis Sorted Set' do
+      expect(user_active_1.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months).to_date)
+      expect(user_active_2.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months).to_date)
+    end
+  end
+
+  describe '#down' do
+    it 'sets last_activity_on to NULL for all users' do
+      mute_stdout { migration.down }
+
+      expect(user_active_1.reload.last_activity_on).to be_nil
+      expect(user_active_2.reload.last_activity_on).to be_nil
+    end
+  end
+
+  def mute_stdout
+    orig_stdout = $stdout
+    $stdout = StringIO.new
+    yield
+    $stdout = orig_stdout
+  end
+end