# frozen_string_literal: true # module Gitlab module Diff class HighlightCache include Gitlab::Utils::Gzip include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection def initialize(diff_collection) @diff_collection = diff_collection end # - Reads from cache # - Assigns DiffFile#highlighted_diff_lines for cached files # def decorate(diff_file) content = read_file(diff_file) return [] unless content # TODO: We could add some kind of flag to #initialize that would allow # us to force re-caching # https://gitlab.com/gitlab-org/gitlab/-/issues/263508 # if content.empty? && recache_due_to_size?(diff_file) # If the file is missing from the cache and there's reason to believe # it is uncached due to a size issue around changing the values for # max patch size, manually populate the hash and then set the value. # new_cache_content = {} new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) write_to_redis_hash(new_cache_content) set_highlighted_diff_lines(diff_file, read_file(diff_file)) else set_highlighted_diff_lines(diff_file, content) end end # For every file that isn't already contained in the redis hash, store the # result of #highlighted_diff_lines, then submit the uncached content # to #write_to_redis_hash to submit a single write. This avoids excessive # IO generated by N+1's (1 writing for each highlighted line or file). # def write_if_empty return if cacheable_files.empty? new_cache_content = {} cacheable_files.each do |diff_file| new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) end write_to_redis_hash(new_cache_content) end def clear Gitlab::Redis::Cache.with do |redis| redis.del(key) end end def key strong_memoize(:redis_key) do [ 'highlighted-diff-files', diffable.cache_key, VERSION, diff_options, Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) ].join(":") end end private def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) end end def recache_due_to_size?(diff_file) diff_file_class = diff_file.diff.class current_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes default_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes(diff_file_class::DEFAULT_MAX_PATCH_BYTES) # If the diff is >= than the default limit, but less than the current # limit, it is likely uncached due to having hit the default limit, # making it eligible for recalculating. # diff_file.diff.diff_bytesize.between?( default_patch_safe_limit_bytes, current_patch_safe_limit_bytes ) end def cacheable_files strong_memoize(:cacheable_files) do diff_files.select { |file| cacheable?(file) && read_file(file).nil? } end end # Given a hash of: # { "file/to/cache" => # [ { line_code: "a5cc2925ca8258af241be7e5b0381edf30266302_19_19", # rich_text: " config/initializers/secret_token.rb\n", # text: " config/initializers/secret_token.rb", # type: nil, # index: 3, # old_pos: 19, # new_pos: 19 } # ] } # # ...it will write/update a Gitlab::Redis hash (HSET) # def write_to_redis_hash(hash) Gitlab::Redis::Cache.with do |redis| redis.pipelined do hash.each do |diff_file_id, highlighted_diff_lines_hash| redis.hset( key, diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) ) end # HSETs have to have their expiration date manually updated # redis.expire(key, EXPIRATION) end record_memory_usage(fetch_memory_usage(redis, key)) end # Subsequent read_file calls would need the latest cache. # clear_memoization(:cached_content) clear_memoization(:cacheable_files) end def record_memory_usage(memory_usage) if memory_usage current_transaction&.observe(:gitlab_redis_diff_caching_memory_usage_bytes, memory_usage) do docstring 'Redis diff caching memory usage by key' buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000] end end end def fetch_memory_usage(redis, key) # Redis versions prior to 4.0.0 do not support memory usage reporting # for a specific key. As of 11-March-2020 we support Redis 3.x, so # need to account for this. We can remove this check once we # officially cease supporting versions <4.0.0. # return if Gem::Version.new(redis.info["redis_version"]) < Gem::Version.new("4") redis.memory("USAGE", key) end def file_paths strong_memoize(:file_paths) do diff_files.collect(&:file_path) end end def read_file(diff_file) cached_content[diff_file.file_path] end def cached_content strong_memoize(:cached_content) { read_cache } end def read_cache return {} unless file_paths.any? results = [] Gitlab::Redis::Cache.with do |redis| results = redis.hmget(key, file_paths) end results.map! do |result| Gitlab::Json.parse(gzip_decompress(result), symbolize_names: true) unless result.nil? end file_paths.zip(results).to_h end def cacheable?(diff_file) diffable.present? && diff_file.text? && diff_file.diffable? end def diff_files # We access raw_diff_files here, as diff_files will attempt to apply the # highlighting code found in this class, leading to a circular # reference. # @diff_collection.raw_diff_files end def current_transaction ::Gitlab::Metrics::Transaction.current end end end end