Merge branch '60383-setup-dashboard-endpoint' into 'master'

Create dashboards endpoint & setup dashboard post-processing

Closes #60383

See merge request gitlab-org/gitlab-ce!27405

app/controllers/projects/environments_controller.rb

@@ -10,8 +10,9 @@ class Projects::EnvironmentsController < Projects::ApplicationController
   before_action :environment, only: [:show, :edit, :update, :stop, :terminal, :terminal_websocket_authorize, :metrics]
   before_action :verify_api_request!, only: :terminal_websocket_authorize
   before_action :expire_etag_cache, only: [:index]
-  before_action only: [:metrics, :additional_metrics] do
+  before_action only: [:metrics, :additional_metrics, :metrics_dashboard] do
     push_frontend_feature_flag(:metrics_time_window)
+    push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint)
   end
 
   def index
@@ -156,6 +157,20 @@ class Projects::EnvironmentsController < Projects::ApplicationController
     end
   end
 
+  def metrics_dashboard
+    return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, @project)
+
+    result = Gitlab::Metrics::Dashboard::Service.new(@project, @current_user, environment: environment).get_dashboard
+
+    respond_to do |format|
+      if result[:status] == :success
+        format.json { render status: :ok, json: result }
+      else
+        format.json { render status: result[:http_status], json: result }
+      end
+    end
+  end
+
   def search
     respond_to do |format|
       format.json do

config/routes/project.rb

@@ -218,6 +218,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           get :terminal
           get :metrics
           get :additional_metrics
+          get :metrics_dashboard
           get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil }
 
           get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy'

lib/gitlab/metrics/dashboard/processor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      # Responsible for processesing a dashboard hash, inserting
+      # relevant DB records & sorting for proper rendering in
+      # the UI. These includes shared metric info, custom metrics
+      # info, and alerts (only in EE).
+      class Processor
+        SEQUENCE = [
+          Stages::CommonMetricsInserter,
+          Stages::ProjectMetricsInserter,
+          Stages::Sorter
+        ].freeze
+
+        def initialize(project, environment, dashboard)
+          @project = project
+          @environment = environment
+          @dashboard = dashboard
+        end
+
+        # Returns a new dashboard hash with the results of
+        # running transforms on the dashboard.
+        def process
+          @dashboard.deep_symbolize_keys.tap do |dashboard|
+            sequence.each do |stage|
+              stage.new(@project, @environment, dashboard).transform!
+            end
+          end
+        end
+
+        private
+
+        def sequence
+          SEQUENCE
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/service.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Fetches the metrics dashboard layout and supplemented the output with DB info.
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Service < ::BaseService
+        SYSTEM_DASHBOARD_NAME = 'common_metrics'
+        SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml")
+
+        # Returns a DB-supplemented json representation of a dashboard config file.
+        def get_dashboard
+          dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard }
+
+          dashboard = process_dashboard(dashboard_string)
+
+          success(dashboard: dashboard)
+        rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e
+          error(e.message, :unprocessable_entity)
+        end
+
+        private
+
+        # Returns the base metrics shipped with every GitLab service.
+        def system_dashboard
+          YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH))
+        end
+
+        def cache_key
+          "metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}"
+        end
+
+        # Returns a new dashboard Hash, supplemented with DB info
+        def process_dashboard(dashboard)
+          Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/stages/base_stage.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class BaseStage
+          DashboardLayoutError = Class.new(StandardError)
+
+          DEFAULT_PANEL_TYPE = 'area-chart'
+
+          attr_reader :project, :environment, :dashboard
+
+          def initialize(project, environment, dashboard)
+            @project = project
+            @environment = environment
+            @dashboard = dashboard
+          end
+
+          # Entry-point to the stage
+          def transform!
+            raise NotImplementedError
+          end
+
+          protected
+
+          def missing_panel_groups!
+            raise DashboardLayoutError.new('Top-level key :panel_groups must be an array')
+          end
+
+          def missing_panels!
+            raise DashboardLayoutError.new('Each "panel_group" must define an array :panels')
+          end
+
+          def missing_metrics!
+            raise DashboardLayoutError.new('Each "panel" must define an array :metrics')
+          end
+
+          def for_metrics(dashboard)
+            missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array)
+
+            dashboard[:panel_groups].each do |panel_group|
+              missing_panels! unless panel_group[:panels].is_a?(Array)
+
+              panel_group[:panels].each do |panel|
+                missing_metrics! unless panel[:metrics].is_a?(Array)
+
+                panel[:metrics].each do |metric|
+                  yield metric
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class CommonMetricsInserter < BaseStage
+          # For each metric in the dashboard config, attempts to
+          # find a corresponding database record. If found,
+          # includes the record's id in the dashboard config.
+          def transform!
+            common_metrics = ::PrometheusMetric.common
+
+            for_metrics(dashboard) do |metric|
+              metric_record = common_metrics.find { |m| m.identifier == metric[:id] }
+              metric[:metric_id] = metric_record.id if metric_record
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class ProjectMetricsInserter < BaseStage
+          # Inserts project-specific metrics into the dashboard
+          # config. If there are no project-specific metrics,
+          # this will have no effect.
+          def transform!
+            project.prometheus_metrics.each do |project_metric|
+              group = find_or_create_panel_group(dashboard[:panel_groups], project_metric)
+              panel = find_or_create_panel(group[:panels], project_metric)
+              find_or_create_metric(panel[:metrics], project_metric)
+            end
+          end
+
+          private
+
+          # Looks for a panel_group corresponding to the
+          # provided metric object. If unavailable, inserts one.
+          # @param panel_groups [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_panel_group(panel_groups, metric)
+            panel_group = find_panel_group(panel_groups, metric)
+            return panel_group if panel_group
+
+            panel_group = new_panel_group(metric)
+            panel_groups << panel_group
+
+            panel_group
+          end
+
+          # Looks for a panel corresponding to the provided
+          # metric object. If unavailable, inserts one.
+          # @param panels [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_panel(panels, metric)
+            panel = find_panel(panels, metric)
+            return panel if panel
+
+            panel = new_panel(metric)
+            panels << panel
+
+            panel
+          end
+
+          # Looks for a metric corresponding to the provided
+          # metric object. If unavailable, inserts one.
+          # @param metrics [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_metric(metrics, metric)
+            target_metric = find_metric(metrics, metric)
+            return target_metric if target_metric
+
+            target_metric = new_metric(metric)
+            metrics << target_metric
+
+            target_metric
+          end
+
+          def find_panel_group(panel_groups, metric)
+            return unless panel_groups
+
+            panel_groups.find { |group| group[:group] == metric.group_title }
+          end
+
+          def find_panel(panels, metric)
+            return unless panels
+
+            panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label]
+            panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers }
+          end
+
+          def find_metric(metrics, metric)
+            return unless metrics
+
+            metrics.find { |m| m[:id] == metric.identifier }
+          end
+
+          def new_panel_group(metric)
+            {
+              group: metric.group_title,
+              priority: metric.priority,
+              panels: []
+            }
+          end
+
+          def new_panel(metric)
+            {
+              type: DEFAULT_PANEL_TYPE,
+              title: metric.title,
+              y_label: metric.y_label,
+              metrics: []
+            }
+          end
+
+          def new_metric(metric)
+            metric.queries.first.merge(metric_id: metric.id)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/dashboard/stages/sorter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class Sorter < BaseStage
+          def transform!
+            missing_panel_groups! unless dashboard[:panel_groups].is_a? Array
+
+            sort_groups!
+            sort_panels!
+          end
+
+          private
+
+          # Sorts the groups in the dashboard by the :priority key
+          def sort_groups!
+            dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i }
+          end
+
+          # Sorts the panels in the dashboard by the :weight key
+          def sort_panels!
+            dashboard[:panel_groups].each do |group|
+              missing_panels! unless group[:panels].is_a? Array
+
+              group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i }
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/controllers/projects/environments_controller_spec.rb

@@ -461,6 +461,43 @@ describe Projects::EnvironmentsController do
     end
   end
 
+  describe 'metrics_dashboard' do
+    context 'when prometheus endpoint is disabled' do
+      before do
+        stub_feature_flags(environment_metrics_use_prometheus_endpoint: false)
+      end
+
+      it 'responds with status code 403' do
+        get :metrics_dashboard, params: environment_params(format: :json)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'when prometheus endpoint is enabled' do
+      it 'returns a json representation of the environment dashboard' do
+        get :metrics_dashboard, params: environment_params(format: :json)
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response.keys).to contain_exactly('dashboard', 'status')
+        expect(json_response['dashboard']).to be_an_instance_of(Hash)
+      end
+
+      context 'when the dashboard could not be provided' do
+        before do
+          allow(YAML).to receive(:safe_load).and_return({})
+        end
+
+        it 'returns an error response' do
+          get :metrics_dashboard, params: environment_params(format: :json)
+
+          expect(response).to have_gitlab_http_status(:unprocessable_entity)
+          expect(json_response.keys).to contain_exactly('message', 'status', 'http_status')
+        end
+      end
+    end
+  end
+
   describe 'GET #search' do
     before do
       create(:environment, name: 'staging', project: project)

spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml

@@ -0,0 +1,36 @@
+dashboard: 'Test Dashboard'
+priority: 1
+panel_groups:
+- group: Group A
+  priority: 10
+  panels:
+  - title: "Super Chart A1"
+    type: "area-chart"
+    y_label: "y_label"
+    weight: 1
+    metrics:
+    - id: metric_a1
+      query_range: 'query'
+      unit: unit
+      label: Legend Label
+  - title: "Super Chart A2"
+    type: "area-chart"
+    y_label: "y_label"
+    weight: 2
+    metrics:
+    - id: metric_a2
+      query_range: 'query'
+      label: Legend Label
+      unit: unit
+- group: Group B
+  priority: 1
+  panels:
+  - title: "Super Chart B"
+    type: "area-chart"
+    y_label: "y_label"
+    weight: 1
+    metrics:
+    - id: metric_b
+      query_range: 'query'
+      unit: unit
+      label: Legend Label

spec/fixtures/lib/gitlab/metrics/dashboard/schemas/dashboard.json

@@ -0,0 +1,13 @@
+{
+  "type": "object",
+  "required": ["dashboard", "priority", "panel_groups"],
+  "properties": {
+    "dashboard": { "type": "string" },
+    "priority": { "type": "number" },
+    "panel_groups": {
+      "type": "array",
+      "items": { "$ref": "spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json" }
+    }
+  },
+  "additionalProperties": false
+}

spec/fixtures/lib/gitlab/metrics/dashboard/schemas/metrics.json

@@ -0,0 +1,20 @@
+{
+  "type": "object",
+  "required": [
+    "unit",
+    "label"
+  ],
+  "oneOf": [
+    { "required": ["query"] },
+    { "required": ["query_range"] }
+  ],
+  "properties": {
+    "id": { "type": "string" },
+    "query_range": { "type": "string" },
+    "query": { "type": "string" },
+    "unit": { "type": "string" },
+    "label": { "type": "string" },
+    "track": { "type": "string" }
+  },
+  "additionalProperties": false
+}

spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json

@@ -0,0 +1,17 @@
+{
+  "type": "object",
+  "required": [
+    "group",
+    "priority",
+    "panels"
+  ],
+  "properties": {
+    "group": { "type": "string" },
+    "priority": { "type": "number" },
+    "panels": {
+      "type": "array",
+      "items": { "$ref": "panels.json" }
+    }
+  },
+  "additionalProperties": false
+}

spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json

@@ -0,0 +1,20 @@
+{
+  "type": "object",
+  "required": [
+    "title",
+    "y_label",
+    "weight",
+    "metrics"
+  ],
+  "properties": {
+    "title": { "type": "string" },
+    "type": { "type": "string" },
+    "y_label": { "type": "string" },
+    "weight": { "type": "number" },
+    "metrics": {
+      "type": "array",
+      "items": { "$ref": "metrics.json" }
+    }
+  },
+  "additionalProperties": false
+}

spec/lib/gitlab/metrics/dashboard/processor_spec.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Metrics::Dashboard::Processor do
+  let(:project) { build(:project) }
+  let(:environment) { build(:environment) }
+  let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
+
+  describe 'process' do
+    let(:process_params) { [project, environment, dashboard_yml] }
+    let(:dashboard) { described_class.new(*process_params).process }
+
+    context 'when dashboard config corresponds to common metrics' do
+      let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') }
+
+      it 'inserts metric ids into the config' do
+        target_metric = all_metrics.find { |metric| metric[:id] == 'metric_a1' }
+
+        expect(target_metric).to include(:metric_id)
+        expect(target_metric[:metric_id]).to eq(common_metric.id)
+      end
+    end
+
+    context 'when the project has associated metrics' do
+      let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) }
+      let!(:project_system_metric) { create(:prometheus_metric, project: project, group: :system) }
+      let!(:project_business_metric) { create(:prometheus_metric, project: project, group: :business) }
+
+      it 'includes project-specific metrics' do
+        expect(all_metrics).to include get_metric_details(project_system_metric)
+        expect(all_metrics).to include get_metric_details(project_response_metric)
+        expect(all_metrics).to include get_metric_details(project_business_metric)
+      end
+
+      it 'orders groups by priority and panels by weight' do
+        expected_metrics_order = [
+          'metric_a2', # group priority 10, panel weight 2
+          'metric_a1', # group priority 10, panel weight 1
+          'metric_b', # group priority 1, panel weight 1
+          project_business_metric.id, # group priority 0, panel weight nil (0)
+          project_response_metric.id, # group priority -5, panel weight nil (0)
+          project_system_metric.id, # group priority -10, panel weight nil (0)
+        ]
+        actual_metrics_order = all_metrics.map { |m| m[:id] || m[:metric_id] }
+
+        expect(actual_metrics_order).to eq expected_metrics_order
+      end
+    end
+
+    shared_examples_for 'errors with message' do |expected_message|
+      it 'raises a DashboardLayoutError' do
+        error_class = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError
+
+        expect { dashboard }.to raise_error(error_class, expected_message)
+      end
+    end
+
+    context 'when the dashboard is missing panel_groups' do
+      let(:dashboard_yml) { {} }
+
+      it_behaves_like 'errors with message', 'Top-level key :panel_groups must be an array'
+    end
+
+    context 'when the dashboard contains a panel_group which is missing panels' do
+      let(:dashboard_yml) { { panel_groups: [{}] } }
+
+      it_behaves_like 'errors with message', 'Each "panel_group" must define an array :panels'
+    end
+
+    context 'when the dashboard contains a panel which is missing metrics' do
+      let(:dashboard_yml) { { panel_groups: [{ panels: [{}] }] } }
+
+      it_behaves_like 'errors with message', 'Each "panel" must define an array :metrics'
+    end
+  end
+
+  private
+
+  def all_metrics
+    dashboard[:panel_groups].map do |group|
+      group[:panels].map { |panel| panel[:metrics] }
+    end.flatten
+  end
+
+  def get_metric_details(metric)
+    {
+      query_range: metric.query,
+      unit: metric.unit,
+      label: metric.legend,
+      metric_id: metric.id
+    }
+  end
+end

spec/lib/gitlab/metrics/dashboard/service_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Metrics::Dashboard::Service, :use_clean_rails_memory_store_caching do
+  let(:project) { build(:project) }
+  let(:environment) { build(:environment) }
+
+  describe 'get_dashboard' do
+    let(:dashboard_schema) { JSON.parse(fixture_file('lib/gitlab/metrics/dashboard/schemas/dashboard.json')) }
+
+    it 'returns a json representation of the environment dashboard' do
+      result = described_class.new(project, environment).get_dashboard
+
+      expect(result.keys).to contain_exactly(:dashboard, :status)
+      expect(result[:status]).to eq(:success)
+
+      expect(JSON::Validator.fully_validate(dashboard_schema, result[:dashboard])).to be_empty
+    end
+
+    it 'caches the dashboard for subsequent calls' do
+      expect(YAML).to receive(:safe_load).once.and_call_original
+
+      described_class.new(project, environment).get_dashboard
+      described_class.new(project, environment).get_dashboard
+    end
+
+    context 'when the dashboard is configured incorrectly' do
+      before do
+        allow(YAML).to receive(:safe_load).and_return({})
+      end
+
+      it 'returns an appropriate message and status code' do
+        result = described_class.new(project, environment).get_dashboard
+
+        expect(result.keys).to contain_exactly(:message, :http_status, :status)
+        expect(result[:status]).to eq(:error)
+        expect(result[:http_status]).to eq(:unprocessable_entity)
+      end
+    end
+  end
+end