2020-05-22 00:08:07 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
# Usage data utilities
|
|
|
|
#
|
|
|
|
# * distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
|
|
|
|
# Does a distinct batch count, smartly reduces batch_size and handles errors
|
|
|
|
#
|
|
|
|
# Examples:
|
|
|
|
# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
|
|
|
|
#
|
|
|
|
# * count(relation, column = nil, batch: true, start: nil, finish: nil)
|
|
|
|
# Does a non-distinct batch count, smartly reduces batch_size and handles errors
|
|
|
|
#
|
|
|
|
# Examples:
|
|
|
|
# active_user_count: count(User.active)
|
|
|
|
#
|
|
|
|
# * alt_usage_data method
|
2020-05-28 00:08:37 +00:00
|
|
|
# handles StandardError and fallbacks by default into -1 this way not all measures fail if we encounter one exception
|
|
|
|
# there might be cases where we need to set a specific fallback in order to be aligned wih what version app is expecting as a type
|
2020-05-22 00:08:07 +00:00
|
|
|
#
|
|
|
|
# Examples:
|
|
|
|
# alt_usage_data { Gitlab::VERSION }
|
|
|
|
# alt_usage_data { Gitlab::CurrentSettings.uuid }
|
2020-05-28 00:08:37 +00:00
|
|
|
# alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled }
|
2020-05-22 00:08:07 +00:00
|
|
|
#
|
|
|
|
# * redis_usage_data method
|
2021-02-24 09:11:09 +00:00
|
|
|
# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent,
|
|
|
|
# Gitlab::UsageDataCounters::HLLRedisCounter::EventError
|
2020-05-22 00:08:07 +00:00
|
|
|
# returns -1 when a block is sent or hash with all values -1 when a counter is sent
|
|
|
|
# different behaviour due to 2 different implementations of redis counter
|
|
|
|
#
|
|
|
|
# Examples:
|
|
|
|
# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
|
|
|
|
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
|
|
|
|
|
|
|
|
module Gitlab
|
|
|
|
module Utils
|
|
|
|
module UsageData
|
2021-04-09 06:09:30 +00:00
|
|
|
include Gitlab::Utils::StrongMemoize
|
2020-05-22 00:08:07 +00:00
|
|
|
extend self
|
|
|
|
|
|
|
|
FALLBACK = -1
|
2021-03-12 18:09:23 +00:00
|
|
|
HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze
|
2020-12-03 15:09:46 +00:00
|
|
|
DISTRIBUTED_HLL_FALLBACK = -2
|
2021-03-08 18:09:12 +00:00
|
|
|
ALL_TIME_TIME_FRAME_NAME = "all"
|
|
|
|
SEVEN_DAYS_TIME_FRAME_NAME = "7d"
|
|
|
|
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d"
|
2021-03-12 18:09:23 +00:00
|
|
|
MAX_BUCKET_SIZE = 100
|
2020-05-22 00:08:07 +00:00
|
|
|
|
2020-09-28 18:09:40 +00:00
|
|
|
def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
|
2020-06-04 15:08:21 +00:00
|
|
|
if batch
|
2020-09-28 18:09:40 +00:00
|
|
|
Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish)
|
2020-05-22 00:08:07 +00:00
|
|
|
else
|
|
|
|
relation.count
|
|
|
|
end
|
|
|
|
rescue ActiveRecord::StatementInvalid
|
|
|
|
FALLBACK
|
|
|
|
end
|
|
|
|
|
2020-06-03 00:08:38 +00:00
|
|
|
def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
|
2020-06-04 15:08:21 +00:00
|
|
|
if batch
|
2020-06-03 00:08:38 +00:00
|
|
|
Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish)
|
2020-05-22 00:08:07 +00:00
|
|
|
else
|
|
|
|
relation.distinct_count_by(column)
|
|
|
|
end
|
|
|
|
rescue ActiveRecord::StatementInvalid
|
|
|
|
FALLBACK
|
|
|
|
end
|
|
|
|
|
2020-12-03 15:09:46 +00:00
|
|
|
def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
|
2021-01-26 21:09:04 +00:00
|
|
|
buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter
|
2020-12-23 12:10:26 +00:00
|
|
|
.new(relation, column)
|
|
|
|
.execute(batch_size: batch_size, start: start, finish: finish)
|
2021-01-26 21:09:04 +00:00
|
|
|
|
|
|
|
yield buckets if block_given?
|
|
|
|
|
|
|
|
buckets.estimated_distinct_count
|
2020-12-03 15:09:46 +00:00
|
|
|
rescue ActiveRecord::StatementInvalid
|
|
|
|
FALLBACK
|
|
|
|
# catch all rescue should be removed as a part of feature flag rollout issue
|
|
|
|
# https://gitlab.com/gitlab-org/gitlab/-/issues/285485
|
|
|
|
rescue StandardError => error
|
|
|
|
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
|
|
|
|
DISTRIBUTED_HLL_FALLBACK
|
|
|
|
end
|
|
|
|
|
2020-09-16 06:09:24 +00:00
|
|
|
def sum(relation, column, batch_size: nil, start: nil, finish: nil)
|
|
|
|
Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish)
|
|
|
|
rescue ActiveRecord::StatementInvalid
|
|
|
|
FALLBACK
|
|
|
|
end
|
|
|
|
|
2021-03-12 18:09:23 +00:00
|
|
|
# We don't support batching with histograms.
|
|
|
|
# Please avoid using this method on large tables.
|
|
|
|
# See https://gitlab.com/gitlab-org/gitlab/-/issues/323949.
|
|
|
|
#
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
|
|
def histogram(relation, column, buckets:, bucket_size: buckets.size)
|
|
|
|
# Using lambda to avoid exposing histogram specific methods
|
|
|
|
parameters_valid = lambda do
|
|
|
|
error_message =
|
|
|
|
if buckets.first == buckets.last
|
|
|
|
'Lower bucket bound cannot equal to upper bucket bound'
|
|
|
|
elsif bucket_size == 0
|
|
|
|
'Bucket size cannot be zero'
|
|
|
|
elsif bucket_size > MAX_BUCKET_SIZE
|
|
|
|
"Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}"
|
|
|
|
end
|
|
|
|
|
|
|
|
return true unless error_message
|
|
|
|
|
|
|
|
exception = ArgumentError.new(error_message)
|
|
|
|
exception.set_backtrace(caller)
|
|
|
|
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception)
|
|
|
|
|
|
|
|
false
|
|
|
|
end
|
|
|
|
|
|
|
|
return HISTOGRAM_FALLBACK unless parameters_valid.call
|
|
|
|
|
|
|
|
count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped'))
|
|
|
|
cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped)
|
|
|
|
|
2021-04-21 12:09:16 +00:00
|
|
|
# For example, 9 segments gives 10 buckets
|
2021-03-12 18:09:23 +00:00
|
|
|
bucket_segments = bucket_size - 1
|
|
|
|
|
|
|
|
width_bucket = Arel::Nodes::NamedFunction
|
|
|
|
.new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments])
|
|
|
|
.as('buckets')
|
|
|
|
|
|
|
|
query = cte
|
|
|
|
.table
|
|
|
|
.project(width_bucket, cte.table[:count])
|
|
|
|
.group('buckets')
|
|
|
|
.order('buckets')
|
|
|
|
.with(cte.to_arel)
|
|
|
|
|
|
|
|
# Return the histogram as a Hash because buckets are unique.
|
|
|
|
relation
|
|
|
|
.connection
|
|
|
|
.exec_query(query.to_sql)
|
|
|
|
.rows
|
|
|
|
.to_h
|
|
|
|
# Keys are converted to strings in Usage Ping JSON
|
|
|
|
.stringify_keys
|
|
|
|
rescue ActiveRecord::StatementInvalid => e
|
|
|
|
Gitlab::AppJsonLogger.error(
|
|
|
|
event: 'histogram',
|
|
|
|
relation: relation.table_name,
|
|
|
|
operation: 'histogram',
|
|
|
|
operation_args: [column, buckets.first, buckets.last, bucket_segments],
|
|
|
|
query: query.to_sql,
|
|
|
|
message: e.message
|
|
|
|
)
|
|
|
|
|
|
|
|
HISTOGRAM_FALLBACK
|
|
|
|
end
|
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
|
2021-02-25 09:10:45 +00:00
|
|
|
def add(*args)
|
|
|
|
return -1 if args.any?(&:negative?)
|
|
|
|
|
|
|
|
args.sum
|
|
|
|
rescue StandardError
|
|
|
|
FALLBACK
|
|
|
|
end
|
|
|
|
|
2020-05-22 00:08:07 +00:00
|
|
|
def alt_usage_data(value = nil, fallback: FALLBACK, &block)
|
|
|
|
if block_given?
|
|
|
|
yield
|
|
|
|
else
|
|
|
|
value
|
|
|
|
end
|
2021-04-26 12:09:44 +00:00
|
|
|
rescue StandardError
|
2020-05-22 00:08:07 +00:00
|
|
|
fallback
|
|
|
|
end
|
|
|
|
|
|
|
|
def redis_usage_data(counter = nil, &block)
|
|
|
|
if block_given?
|
|
|
|
redis_usage_counter(&block)
|
|
|
|
elsif counter.present?
|
|
|
|
redis_usage_data_totals(counter)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-03-12 12:09:33 +00:00
|
|
|
def with_prometheus_client(fallback: {}, verify: true)
|
2020-09-16 18:09:47 +00:00
|
|
|
client = prometheus_client(verify: verify)
|
|
|
|
return fallback unless client
|
2020-07-02 00:09:51 +00:00
|
|
|
|
2020-09-16 18:09:47 +00:00
|
|
|
yield client
|
2021-04-26 12:09:44 +00:00
|
|
|
rescue StandardError
|
2021-03-12 12:09:33 +00:00
|
|
|
fallback
|
2020-06-03 12:08:21 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def measure_duration
|
|
|
|
result = nil
|
|
|
|
duration = Benchmark.realtime do
|
|
|
|
result = yield
|
|
|
|
end
|
|
|
|
[result, duration]
|
|
|
|
end
|
|
|
|
|
2020-06-17 12:08:42 +00:00
|
|
|
def with_finished_at(key, &block)
|
2020-08-19 06:10:04 +00:00
|
|
|
yield.merge(key => Time.current)
|
2020-06-17 12:08:42 +00:00
|
|
|
end
|
|
|
|
|
2020-09-18 03:09:28 +00:00
|
|
|
# @param event_name [String] the event name
|
|
|
|
# @param values [Array|String] the values counted
|
|
|
|
def track_usage_event(event_name, values)
|
2021-01-08 00:10:44 +00:00
|
|
|
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
|
2020-09-10 21:08:28 +00:00
|
|
|
end
|
|
|
|
|
2021-04-21 15:09:35 +00:00
|
|
|
def maximum_id(model, column = nil)
|
|
|
|
key = :"#{model.name.downcase.gsub('::', '_')}_maximum_id"
|
|
|
|
column_to_read = column || :id
|
|
|
|
|
2021-04-09 06:09:30 +00:00
|
|
|
strong_memoize(key) do
|
2021-04-21 15:09:35 +00:00
|
|
|
model.maximum(column_to_read)
|
2021-04-09 06:09:30 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-04-21 15:09:35 +00:00
|
|
|
# rubocop: disable UsageData/LargeTable:
|
|
|
|
def jira_service_data
|
|
|
|
data = {
|
|
|
|
projects_jira_server_active: 0,
|
|
|
|
projects_jira_cloud_active: 0
|
|
|
|
}
|
|
|
|
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
2021-05-22 00:10:27 +00:00
|
|
|
::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services|
|
2021-04-21 15:09:35 +00:00
|
|
|
counts = services.group_by do |service|
|
|
|
|
# TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
|
|
|
|
service_url = service.data_fields&.url || (service.properties && service.properties['url'])
|
|
|
|
service_url&.include?('.atlassian.net') ? :cloud : :server
|
|
|
|
end
|
|
|
|
|
|
|
|
data[:projects_jira_server_active] += counts[:server].size if counts[:server]
|
|
|
|
data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud]
|
|
|
|
end
|
|
|
|
|
|
|
|
data
|
|
|
|
end
|
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
# rubocop: enable UsageData/LargeTable:
|
|
|
|
|
|
|
|
def minimum_id(model, column = nil)
|
|
|
|
key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id"
|
|
|
|
column_to_read = column || :id
|
|
|
|
|
2021-04-09 06:09:30 +00:00
|
|
|
strong_memoize(key) do
|
2021-04-21 15:09:35 +00:00
|
|
|
model.minimum(column_to_read)
|
2021-04-09 06:09:30 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-04-21 15:09:35 +00:00
|
|
|
def epics_deepest_relationship_level
|
|
|
|
# rubocop: disable UsageData/LargeTable
|
|
|
|
{ epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i }
|
|
|
|
# rubocop: enable UsageData/LargeTable
|
|
|
|
end
|
|
|
|
|
2020-05-22 00:08:07 +00:00
|
|
|
private
|
|
|
|
|
2020-09-16 18:09:47 +00:00
|
|
|
def prometheus_client(verify:)
|
|
|
|
server_address = prometheus_server_address
|
|
|
|
|
|
|
|
return unless server_address
|
|
|
|
|
|
|
|
# There really is not a way to discover whether a Prometheus connection is using TLS or not
|
|
|
|
# Try TLS first because HTTPS will return fast if failed.
|
|
|
|
%w[https http].find do |scheme|
|
|
|
|
api_url = "#{scheme}://#{server_address}"
|
|
|
|
client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify)
|
|
|
|
break client if client.ready?
|
2021-04-26 12:09:44 +00:00
|
|
|
rescue StandardError
|
2020-09-16 18:09:47 +00:00
|
|
|
nil
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def prometheus_server_address
|
2020-09-11 21:08:44 +00:00
|
|
|
if Gitlab::Prometheus::Internal.prometheus_enabled?
|
2021-01-05 09:10:15 +00:00
|
|
|
# Stripping protocol from URI
|
|
|
|
Gitlab::Prometheus::Internal.uri&.strip&.sub(%r{^https?://}, '')
|
2020-09-11 21:08:44 +00:00
|
|
|
elsif Gitlab::Consul::Internal.api_url
|
2020-09-16 18:09:47 +00:00
|
|
|
Gitlab::Consul::Internal.discover_prometheus_server_address
|
2020-09-11 21:08:44 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-05-22 00:08:07 +00:00
|
|
|
def redis_usage_counter
|
|
|
|
yield
|
2021-02-24 09:11:09 +00:00
|
|
|
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError
|
2020-05-22 00:08:07 +00:00
|
|
|
FALLBACK
|
|
|
|
end
|
|
|
|
|
|
|
|
def redis_usage_data_totals(counter)
|
|
|
|
counter.totals
|
|
|
|
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
|
|
|
|
counter.fallback_totals
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|