From 2061414054ce43aa6d53d1be3f602114e5a336d2 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 10 May 2017 11:25:30 +0200 Subject: [PATCH] Additional metrics initial work, with working metrics listing, but without actoual metrics mesurements --- .../projects/environments_controller.rb | 6 ++ .../projects/prometheus_controller.rb | 22 ++++++ app/models/environment.rb | 8 +++ .../project_services/prometheus_service.rb | 20 ++++-- config/additional_metrics.yml | 26 +++++++ config/routes/project.rb | 5 ++ lib/gitlab/prometheus/metric.rb | 28 ++++++++ lib/gitlab/prometheus/metric_group.rb | 33 +++++++++ lib/gitlab/prometheus/metrics_sources.rb | 7 ++ lib/gitlab/prometheus/parsing_error.rb | 3 + .../queries/additional_metrics_query.rb | 70 ++++++++++++++++++ lib/gitlab/prometheus/queries/base_query.rb | 2 +- .../queries/matched_metrics_query.rb | 72 +++++++++++++++++++ lib/gitlab/prometheus_client.rb | 8 +++ 14 files changed, 304 insertions(+), 6 deletions(-) create mode 100644 app/controllers/projects/prometheus_controller.rb create mode 100644 config/additional_metrics.yml create mode 100644 lib/gitlab/prometheus/metric.rb create mode 100644 lib/gitlab/prometheus/metric_group.rb create mode 100644 lib/gitlab/prometheus/metrics_sources.rb create mode 100644 lib/gitlab/prometheus/parsing_error.rb create mode 100644 lib/gitlab/prometheus/queries/additional_metrics_query.rb create mode 100644 lib/gitlab/prometheus/queries/matched_metrics_query.rb diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index efe83776834..6d230e84ef7 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -129,6 +129,12 @@ class Projects::EnvironmentsController < Projects::ApplicationController end end + def additional_metrics + additional_metrics = environment.additional_metrics || {} + + render json: additional_metrics, status: additional_metrics.any? ? :ok : :no_content + end + private def verify_api_request! diff --git a/app/controllers/projects/prometheus_controller.rb b/app/controllers/projects/prometheus_controller.rb new file mode 100644 index 00000000000..74e247535d5 --- /dev/null +++ b/app/controllers/projects/prometheus_controller.rb @@ -0,0 +1,22 @@ +class Projects::PrometheusController < Projects::ApplicationController + before_action :authorize_read_project! + + def active_metrics + return render_404 unless has_prometheus_metrics? + matched_metrics = prometheus_service.reactive_query(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself) + + if matched_metrics + render json: matched_metrics, status: :ok + else + head :no_content + end + end + + def prometheus_service + project.monitoring_service + end + + def has_prometheus_metrics? + prometheus_service&.respond_to?(:reactive_query) + end +end diff --git a/app/models/environment.rb b/app/models/environment.rb index 61572d8d69a..b4a4f74a8d5 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -149,10 +149,18 @@ class Environment < ActiveRecord::Base project.monitoring_service.present? && available? && last_deployment.present? end + def has_additional_metrics? + has_metrics? && project.monitoring_service&.respond_to?(:reactive_query) + end + def metrics project.monitoring_service.environment_metrics(self) if has_metrics? end + def additional_metrics + project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery, self.id) if has_additional_metrics? + end + # An environment name is not necessarily suitable for use in URLs, DNS # or other third-party contexts, so provide a slugified version. A slug has # the following properties: diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index ec72cb6856d..674d485a03c 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -64,23 +64,26 @@ class PrometheusService < MonitoringService end def environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &:itself) + with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics)) end def deployment_metrics(deployment) - metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &:itself) + metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics)) metrics&.merge(deployment_time: created_at.to_i) || {} end + def reactive_query(query_class, *args, &block) + calculate_reactive_cache(query_class, *args, &block) + end + # Cache metrics for specific environment def calculate_reactive_cache(query_class_name, *args) return unless active? && project && !project.pending_delete? - metrics = Kernel.const_get(query_class_name).new(client).query(*args) - + data = Kernel.const_get(query_class_name).new(client).query(*args) { success: true, - metrics: metrics, + data: data, last_update: Time.now.utc } rescue Gitlab::PrometheusError => err @@ -90,4 +93,11 @@ class PrometheusService < MonitoringService def client @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url) end + + private + + def rename_data_to_metrics(metrics) + metrics[:metrics] = metrics.delete :data + metrics + end end diff --git a/config/additional_metrics.yml b/config/additional_metrics.yml new file mode 100644 index 00000000000..209209f4b30 --- /dev/null +++ b/config/additional_metrics.yml @@ -0,0 +1,26 @@ +- group: Kubernetes + priority: 1 + metrics: + - title: "Memory usage" + detect: container_memory_usage_bytes + weight: 1 + queries: + - query_range: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20' + label: Container memory + unit: MiB + - title: "Current memory usage" + detect: container_memory_usage_bytes + weight: 1 + queries: + - query: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20' + unit: MiB + - title: "CPU usage" + detect: container_cpu_usage_seconds_total + weight: 1 + queries: + - query_range: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100' + - title: "Current CPU usage" + detect: container_cpu_usage_seconds_total + weight: 1 + queries: + - query: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100' diff --git a/config/routes/project.rb b/config/routes/project.rb index 9fe8372edf9..f5d00e4e93b 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -72,6 +72,10 @@ constraints(ProjectUrlConstrainer.new) do resource :mattermost, only: [:new, :create] + namespace :prometheus do + get :active_metrics + end + resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create] do member do put :enable @@ -152,6 +156,7 @@ constraints(ProjectUrlConstrainer.new) do post :stop get :terminal get :metrics + get :additional_metrics get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil } end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb new file mode 100644 index 00000000000..2818afb34b0 --- /dev/null +++ b/lib/gitlab/prometheus/metric.rb @@ -0,0 +1,28 @@ +module Gitlab::Prometheus + class Metric + attr_reader :group, :title, :detect, :weight, :queries + + def initialize(group, title, detect, weight, queries = []) + @group = group + @title = title + @detect = detect + @weight = weight + @queries = queries + end + + def self.metric_from_entry(group, entry) + missing_fields = [:title, :detect, :weight, :queries].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + Metric.new(group, entry[:title], entry[:detect], entry[:weight], entry[:queries]) + end + + def self.metrics_from_list(group, list) + list.map { |entry| metric_from_entry(group, entry) } + end + + def self.additional_metrics_raw + @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys) + end + end +end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb new file mode 100644 index 00000000000..093390b4fa7 --- /dev/null +++ b/lib/gitlab/prometheus/metric_group.rb @@ -0,0 +1,33 @@ +module Gitlab::Prometheus + class MetricGroup + attr_reader :priority, :name + attr_accessor :metrics + + def initialize(name, priority, metrics = []) + @name = name + @priority = priority + @metrics = metrics + end + + def self.all + load_groups_from_yaml + end + + def self.group_from_entry(entry) + missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + group = MetricGroup.new(entry[:group], entry[:priority]) + group.metrics = Metric.metrics_from_list(group, entry[:metrics]) + group + end + + def self.load_groups_from_yaml + additional_metrics_raw.map(&method(:group_from_entry)) + end + + def self.additional_metrics_raw + @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze + end + end +end diff --git a/lib/gitlab/prometheus/metrics_sources.rb b/lib/gitlab/prometheus/metrics_sources.rb new file mode 100644 index 00000000000..500b6e971a2 --- /dev/null +++ b/lib/gitlab/prometheus/metrics_sources.rb @@ -0,0 +1,7 @@ +module Gitlab::Prometheus + module MetricsSources + def self.additional_metrics + @additional_metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).deep_symbolize_keys.freeze + end + end +end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb new file mode 100644 index 00000000000..067ea7f878a --- /dev/null +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -0,0 +1,3 @@ +module Gitlab::Prometheus + ParsingError = Class.new(StandardError) +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb new file mode 100644 index 00000000000..001701383c3 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_query.rb @@ -0,0 +1,70 @@ +module Gitlab::Prometheus::Queries + class AdditionalMetricsQuery < BaseQuery + def self.metrics + @metrics ||= YAML.load_file(Rails.root.join('config/custom_metrics.yml')).freeze + end + + def query(environment_id) + environment = Environment.find_by(id: environment_id) + + context = { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + } + + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + matched_metrics.map do |group| + group[:metrics].map! do |metric| + metric[:queries].map! do |query| + query = query.symbolize_keys + query[:result] = + if query.has_key?(:query_range) + client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end) + else + client_query(query[:query] % context, time: timeframe_end) + end + query + end + metric + end + group + end + end + + def process_query(group, query) + result = if query.has_key?(:query_range) + client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end) + else + client_query(query[:query] % context, time: timeframe_end) + end + contains_metrics = result.all? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_result(query_result) + contains_metrics = query_result.all? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + + contains_metrics + end + + def matched_metrics + label_values = client_label_values || [] + + result = Gitlab::Prometheus::MetricsSources.additional_metrics.map do |group| + group[:metrics].map!(&:symbolize_keys) + group[:metrics].select! do |metric| + matcher = Regexp.compile(metric[:detect]) + label_values.any? &matcher.method(:match) + end + group + end + + result.select {|group| !group[:metrics].empty?} + end + end +end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 2a2eb4ae57f..c60828165bd 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client - delegate :query_range, :query, to: :client, prefix: true + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb new file mode 100644 index 00000000000..61926320e40 --- /dev/null +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -0,0 +1,72 @@ +module Gitlab::Prometheus::Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze + + def self.metrics + @metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys) + end + + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end + + def groups_data + metrics_series = metrics_with_series(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_series) + + groups = {} + + metrics_series.each do |metrics, series| + groups[metrics.group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + group = groups[metrics.group] + + if series.all?(&lookup.method(:has_key?)) + group[:active_metrics] += 1 + else + group[:metrics_missing_requirements] += 1 + end + group + end + + groups + end + + def active_series_lookup(metrics) + timeframe_start = 8.hours.ago + timeframe_end = Time.now + + series = metrics.flat_map { |metrics, series| series }.uniq + + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end + + def has_matching_label(series_info) + series_info.has_key?('environment') + end + + def metrics_with_series(metric_groups) + label_values = client_label_values || [] + + metrics = metric_groups.flat_map do |group| + group.metrics.map do |metric| + matcher = Regexp.compile(metric.detect) + [metric, label_values.select(&matcher.method(:match))] + end + end + + metrics.select { |metric, labels| labels&.any? } + end + end +end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 5b51a1779dd..f4ef4ff8ba4 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -29,6 +29,14 @@ module Gitlab end end + def label_values(name='__name__') + json_api_get("label/#{name}/values") + end + + def series(*matches, start: 8.hours.ago, stop: Time.now) + json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) + end + private def json_api_get(type, args = {})