Merge branch 'prom-api-3' into 'master'

Switch charts to Prometheus API endpoint CE-3

Closes #58516

See merge request gitlab-org/gitlab-ce!29280

app/assets/javascripts/monitoring/components/charts/area.vue

@@ -227,6 +227,7 @@ export default {
       [this.primaryColor] = chart.getOption().color;
     },
     onResize() {
+      if (!this.$refs.areaChart) return;
       const { width } = this.$refs.areaChart.$el.getBoundingClientRect();
       this.width = width;
     },

app/assets/javascripts/monitoring/components/dashboard.vue

@@ -137,7 +137,12 @@ export default {
       'showEmptyState',
       'environments',
       'deploymentData',
+      'metricsWithData',
+      'useDashboardEndpoint',
     ]),
+    groupsWithData() {
+      return this.groups.filter(group => this.chartsWithData(group.metrics).length > 0);
+    },
   },
   created() {
     this.setEndpoints({
@@ -182,7 +187,16 @@ export default {
       'fetchData',
       'setGettingStartedEmptyState',
       'setEndpoints',
+      'setDashboardEnabled',
     ]),
+    chartsWithData(charts) {
+      if (!this.useDashboardEndpoint) {
+        return charts;
+      }
+      return charts.filter(chart =>
+        chart.metrics.some(metric => this.metricsWithData.includes(metric.metric_id)),
+      );
+    },
     getGraphAlerts(queries) {
       if (!this.allAlerts) return {};
       const metricIdsForChart = queries.map(q => q.metricId);
@@ -308,13 +322,13 @@ export default {
       </div>
     </div>
     <graph-group
-      v-for="(groupData, index) in groups"
+      v-for="(groupData, index) in groupsWithData"
       :key="index"
       :name="groupData.group"
       :show-panels="showPanels"
     >
       <monitor-area-chart
-        v-for="(graphData, graphIndex) in groupData.metrics"
+        v-for="(graphData, graphIndex) in chartsWithData(groupData.metrics)"
         :key="graphIndex"
         :graph-data="graphData"
         :deployment-data="deploymentData"

app/assets/javascripts/monitoring/stores/actions.js

@@ -42,8 +42,9 @@ export const setDashboardEnabled = ({ commit }, enabled) => {
 export const requestMetricsDashboard = ({ commit }) => {
   commit(types.REQUEST_METRICS_DATA);
 };
-export const receiveMetricsDashboardSuccess = ({ commit }, { response }) => {
+export const receiveMetricsDashboardSuccess = ({ commit, dispatch }, { response, params }) => {
   commit(types.RECEIVE_METRICS_DATA_SUCCESS, response.dashboard.panel_groups);
+  dispatch('fetchPrometheusMetrics', params);
 };
 export const receiveMetricsDashboardFailure = ({ commit }, error) => {
   commit(types.RECEIVE_METRICS_DATA_FAILURE, error);
@@ -98,7 +99,7 @@ export const fetchDashboard = ({ state, dispatch }, params) => {
     .get(state.dashboardEndpoint, { params })
     .then(resp => resp.data)
     .then(response => {
-      dispatch('receiveMetricsDashboardSuccess', { response });
+      dispatch('receiveMetricsDashboardSuccess', { response, params });
     })
     .catch(error => {
       dispatch('receiveMetricsDashboardFailure', error);
@@ -106,6 +107,62 @@ export const fetchDashboard = ({ state, dispatch }, params) => {
     });
 };
 
+function fetchPrometheusResult(prometheusEndpoint, params) {
+  return backOffRequest(() => axios.get(prometheusEndpoint, { params }))
+    .then(res => res.data)
+    .then(response => {
+      if (response.status === 'error') {
+        throw new Error(response.error);
+      }
+
+      return response.data.result;
+    });
+}
+
+/**
+ * Returns list of metrics in data.result
+ * {"status":"success", "data":{"resultType":"matrix","result":[]}}
+ *
+ * @param {metric} metric
+ */
+export const fetchPrometheusMetric = ({ commit }, { metric, params }) => {
+  const { start, end } = params;
+  const timeDiff = end - start;
+
+  const minStep = 60;
+  const queryDataPoints = 600;
+  const step = Math.max(minStep, Math.ceil(timeDiff / queryDataPoints));
+
+  const queryParams = {
+    start,
+    end,
+    step,
+  };
+
+  return fetchPrometheusResult(metric.prometheus_endpoint_path, queryParams).then(result => {
+    commit(types.SET_QUERY_RESULT, { metricId: metric.metric_id, result });
+  });
+};
+
+export const fetchPrometheusMetrics = ({ state, commit, dispatch }, params) => {
+  commit(types.REQUEST_METRICS_DATA);
+
+  const promises = [];
+  state.groups.forEach(group => {
+    group.panels.forEach(panel => {
+      panel.metrics.forEach(metric => {
+        promises.push(dispatch('fetchPrometheusMetric', { metric, params }));
+      });
+    });
+  });
+
+  return Promise.all(promises).then(() => {
+    if (state.metricsWithData.length === 0) {
+      commit(types.SET_NO_DATA_EMPTY_STATE);
+    }
+  });
+};
+
 export const fetchDeploymentsData = ({ state, dispatch }) => {
   if (!state.deploymentEndpoint) {
     return Promise.resolve([]);

app/assets/javascripts/monitoring/stores/mutation_types.js

@@ -7,7 +7,9 @@ export const RECEIVE_DEPLOYMENTS_DATA_FAILURE = 'RECEIVE_DEPLOYMENTS_DATA_FAILUR
 export const REQUEST_ENVIRONMENTS_DATA = 'REQUEST_ENVIRONMENTS_DATA';
 export const RECEIVE_ENVIRONMENTS_DATA_SUCCESS = 'RECEIVE_ENVIRONMENTS_DATA_SUCCESS';
 export const RECEIVE_ENVIRONMENTS_DATA_FAILURE = 'RECEIVE_ENVIRONMENTS_DATA_FAILURE';
+export const SET_QUERY_RESULT = 'SET_QUERY_RESULT';
 export const SET_TIME_WINDOW = 'SET_TIME_WINDOW';
 export const SET_DASHBOARD_ENABLED = 'SET_DASHBOARD_ENABLED';
 export const SET_ENDPOINTS = 'SET_ENDPOINTS';
 export const SET_GETTING_STARTED_EMPTY_STATE = 'SET_GETTING_STARTED_EMPTY_STATE';
+export const SET_NO_DATA_EMPTY_STATE = 'SET_NO_DATA_EMPTY_STATE';

app/assets/javascripts/monitoring/stores/mutations.js

@@ -1,5 +1,6 @@
+import Vue from 'vue';
 import * as types from './mutation_types';
-import { normalizeMetrics, sortMetrics } from './utils';
+import { normalizeMetrics, sortMetrics, normalizeQueryResult } from './utils';
 
 export default {
   [types.REQUEST_METRICS_DATA](state) {
@@ -48,6 +49,26 @@ export default {
   [types.RECEIVE_ENVIRONMENTS_DATA_FAILURE](state) {
     state.environments = [];
   },
+  [types.SET_QUERY_RESULT](state, { metricId, result }) {
+    if (!metricId || !result || result.length === 0) {
+      return;
+    }
+
+    state.showEmptyState = false;
+
+    state.groups.forEach(group => {
+      group.metrics.forEach(metric => {
+        metric.queries.forEach(query => {
+          if (query.metric_id === metricId) {
+            state.metricsWithData.push(metricId);
+            // ensure dates/numbers are correctly formatted for charts
+            const normalizedResults = result.map(normalizeQueryResult);
+            Vue.set(query, 'result', Object.freeze(normalizedResults));
+          }
+        });
+      });
+    });
+  },
   [types.SET_ENDPOINTS](state, endpoints) {
     state.metricsEndpoint = endpoints.metricsEndpoint;
     state.environmentsEndpoint = endpoints.environmentsEndpoint;
@@ -60,4 +81,8 @@ export default {
   [types.SET_GETTING_STARTED_EMPTY_STATE](state) {
     state.emptyState = 'gettingStarted';
   },
+  [types.SET_NO_DATA_EMPTY_STATE](state) {
+    state.showEmptyState = true;
+    state.emptyState = 'noData';
+  },
 };

app/assets/javascripts/monitoring/stores/state.js

@@ -1,14 +1,17 @@
+import invalidUrl from '~/lib/utils/invalid_url';
+
 export default () => ({
   hasMetrics: false,
   showPanels: true,
   metricsEndpoint: null,
   environmentsEndpoint: null,
   deploymentsEndpoint: null,
-  dashboardEndpoint: null,
+  dashboardEndpoint: invalidUrl,
   useDashboardEndpoint: false,
   emptyState: 'gettingStarted',
   showEmptyState: true,
   groups: [],
   deploymentData: [],
   environments: [],
+  metricsWithData: [],
 });

app/assets/javascripts/monitoring/stores/utils.js

@@ -58,6 +58,14 @@ export const sortMetrics = metrics =>
     .sortBy('weight')
     .value();
 
+export const normalizeQueryResult = timeSeries => ({
+  ...timeSeries,
+  values: timeSeries.values.map(([timestamp, value]) => [
+    new Date(timestamp * 1000).toISOString(),
+    Number(value),
+  ]),
+});
+
 export const normalizeMetrics = metrics => {
   const groupedMetrics = groupQueriesByChartInfo(metrics);
 
@@ -66,13 +74,7 @@ export const normalizeMetrics = metrics => {
       ...query,
       // custom metrics do not require a label, so we should ensure this attribute is defined
       label: query.label || metric.y_label,
-      result: (query.result || []).map(timeSeries => ({
-        ...timeSeries,
-        values: timeSeries.values.map(([timestamp, value]) => [
-          new Date(timestamp * 1000).toISOString(),
-          Number(value),
-        ]),
-      })),
+      result: (query.result || []).map(normalizeQueryResult),
     }));
 
     return {

spec/javascripts/monitoring/mock_data.js

@@ -880,6 +880,7 @@ export const metricsDashboardResponse = {
                 label: 'Total',
                 unit: 'GB',
                 metric_id: 12,
+                prometheus_endpoint_path: 'http://test',
               },
             ],
           },

spec/javascripts/monitoring/store/actions_spec.js

@@ -8,6 +8,8 @@ import {
   receiveMetricsDashboardFailure,
   fetchDeploymentsData,
   fetchEnvironmentsData,
+  fetchPrometheusMetrics,
+  fetchPrometheusMetric,
   requestMetricsData,
   setEndpoints,
   setGettingStartedEmptyState,
@@ -15,7 +17,12 @@ import {
 import storeState from '~/monitoring/stores/state';
 import testAction from 'spec/helpers/vuex_action_helper';
 import { resetStore } from '../helpers';
-import { deploymentData, environmentData, metricsDashboardResponse } from '../mock_data';
+import {
+  deploymentData,
+  environmentData,
+  metricsDashboardResponse,
+  metricsGroupsAPIResponse,
+} from '../mock_data';
 
 describe('Monitoring store actions', () => {
   let mock;
@@ -179,6 +186,7 @@ describe('Monitoring store actions', () => {
           expect(dispatch).toHaveBeenCalledWith('requestMetricsDashboard');
           expect(dispatch).toHaveBeenCalledWith('receiveMetricsDashboardSuccess', {
             response,
+            params,
           });
           done();
         })
@@ -220,6 +228,8 @@ describe('Monitoring store actions', () => {
         types.RECEIVE_METRICS_DATA_SUCCESS,
         metricsDashboardResponse.dashboard.panel_groups,
       );
+
+      expect(dispatch).toHaveBeenCalledWith('fetchPrometheusMetrics', params);
     });
   });
 
@@ -242,4 +252,71 @@ describe('Monitoring store actions', () => {
       expect(commit).toHaveBeenCalledWith(types.RECEIVE_METRICS_DATA_FAILURE, 'uh-oh');
     });
   });
+
+  describe('fetchPrometheusMetrics', () => {
+    let commit;
+    let dispatch;
+
+    beforeEach(() => {
+      commit = jasmine.createSpy();
+      dispatch = jasmine.createSpy();
+    });
+
+    it('commits empty state when state.groups is empty', done => {
+      const state = storeState();
+      const params = {};
+
+      fetchPrometheusMetrics({ state, commit, dispatch }, params)
+        .then(() => {
+          expect(commit).toHaveBeenCalledWith(types.SET_NO_DATA_EMPTY_STATE);
+          expect(dispatch).not.toHaveBeenCalled();
+          done();
+        })
+        .catch(done.fail);
+    });
+
+    it('dispatches fetchPrometheusMetric for each panel query', done => {
+      const params = {};
+      const state = storeState();
+      state.groups = metricsDashboardResponse.dashboard.panel_groups;
+
+      const metric = state.groups[0].panels[0].metrics[0];
+
+      fetchPrometheusMetrics({ state, commit, dispatch }, params)
+        .then(() => {
+          expect(dispatch.calls.count()).toEqual(3);
+          expect(dispatch).toHaveBeenCalledWith('fetchPrometheusMetric', { metric, params });
+          done();
+        })
+        .catch(done.fail);
+
+      done();
+    });
+  });
+
+  describe('fetchPrometheusMetric', () => {
+    it('commits prometheus query result', done => {
+      const commit = jasmine.createSpy();
+      const params = {
+        start: '1557216349.469',
+        end: '1557218149.469',
+      };
+      const metric = metricsDashboardResponse.dashboard.panel_groups[0].panels[0].metrics[0];
+      const state = storeState();
+
+      const data = metricsGroupsAPIResponse.data[0].metrics[0].queries[0];
+      const response = { data };
+      mock.onGet('http://test').reply(200, response);
+
+      fetchPrometheusMetric({ state, commit }, { metric, params });
+
+      setTimeout(() => {
+        expect(commit).toHaveBeenCalledWith(types.SET_QUERY_RESULT, {
+          metricId: metric.metric_id,
+          result: data.result,
+        });
+        done();
+      });
+    });
+  });
 });

spec/javascripts/monitoring/store/mutations_spec.js

@@ -118,4 +118,42 @@ describe('Monitoring mutations', () => {
       expect(stateCopy.dashboardEndpoint).toEqual('dashboard.json');
     });
   });
+
+  describe('SET_QUERY_RESULT', () => {
+    const metricId = 12;
+    const result = [{ values: [[0, 1], [1, 1], [1, 3]] }];
+
+    beforeEach(() => {
+      stateCopy.useDashboardEndpoint = true;
+      const dashboardGroups = metricsDashboardResponse.dashboard.panel_groups;
+      mutations[types.RECEIVE_METRICS_DATA_SUCCESS](stateCopy, dashboardGroups);
+    });
+
+    it('clears empty state', () => {
+      mutations[types.SET_QUERY_RESULT](stateCopy, {
+        metricId,
+        result,
+      });
+
+      expect(stateCopy.showEmptyState).toBe(false);
+    });
+
+    it('sets metricsWithData value', () => {
+      mutations[types.SET_QUERY_RESULT](stateCopy, {
+        metricId,
+        result,
+      });
+
+      expect(stateCopy.metricsWithData).toEqual([12]);
+    });
+
+    it('does not store empty results', () => {
+      mutations[types.SET_QUERY_RESULT](stateCopy, {
+        metricId,
+        result: [],
+      });
+
+      expect(stateCopy.metricsWithData).toEqual([]);
+    });
+  });
 });