# frozen_string_literal: true module Database class ConsistencyCheckService CURSOR_REDIS_KEY_TTL = 7.days EMPTY_RESULT = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [] }.freeze def initialize(source_model:, target_model:, source_columns:, target_columns:) @source_model = source_model @target_model = target_model @source_columns = source_columns @target_columns = target_columns @source_sort_column = source_columns.first @target_sort_column = target_columns.first end # This class takes two ActiveRecord models, and compares the selected columns # of the two models tables, for the purposes of checking the consistency of # mirroring of tables. For example Namespace and Ci::NamepaceMirror # # It compares up to 25 batches (1000 records / batch), or up to 30 seconds # for all the batches in total. # # It saves the cursor of the next start_id (cusror) in Redis. If the start_id # wasn't saved in Redis, for example, in the first run, it will choose some random start_id # # Example: # service = Database::ConsistencyCheckService.new( # source_model: Namespace, # target_model: Ci::NamespaceMirror, # source_columns: %w[id traversal_ids], # target_columns: %w[namespace_id traversal_ids], # ) # result = service.execute # # result is a hash that has the following fields: # - batches: Number of batches checked # - matches: The number of matched records # - mismatches: The number of mismatched records # - mismatches_details: It's an array that contains details about the mismatched records. # each record in this array is a hash of format {id: ID, source_table: [...], target_table: [...]} # Each record represents the attributes of the records in the two tables. # - start_id: The start id cursor of the current batch. means no records. # - next_start_id: The ID that can be used for the next batch iteration check. means no records def execute start_id = next_start_id return EMPTY_RESULT if start_id.nil? result = consistency_checker.execute(start_id: start_id) result[:start_id] = start_id save_next_start_id(result[:next_start_id]) result end private attr_reader :source_model, :target_model, :source_columns, :target_columns, :source_sort_column, :target_sort_column def consistency_checker @consistency_checker ||= Gitlab::Database::ConsistencyChecker.new( source_model: source_model, target_model: target_model, source_columns: source_columns, target_columns: target_columns ) end def next_start_id return if min_id.nil? fetch_next_start_id || random_start_id end # rubocop: disable CodeReuse/ActiveRecord def min_id @min_id ||= source_model.minimum(source_sort_column) end def max_id @max_id ||= source_model.minimum(source_sort_column) end # rubocop: enable CodeReuse/ActiveRecord def fetch_next_start_id Gitlab::Redis::SharedState.with { |redis| redis.get(cursor_redis_shared_state_key)&.to_i } end # This returns some random start_id, so that we don't always start checking # from the start of the table, in case we lose the cursor in Redis. def random_start_id range_start = min_id range_end = [min_id, max_id - Gitlab::Database::ConsistencyChecker::BATCH_SIZE].max rand(range_start..range_end) end def save_next_start_id(start_id) Gitlab::Redis::SharedState.with do |redis| redis.set(cursor_redis_shared_state_key, start_id, ex: CURSOR_REDIS_KEY_TTL) end end def cursor_redis_shared_state_key "consistency_check_cursor:#{source_model.table_name}:#{target_model.table_name}" end end end