wip

app/models/clusters/applications/prometheus.rb

@@ -14,6 +14,18 @@ module Clusters
         'stable/prometheus'
       end
 
+      def namespace
+        Gitlab::Kubernetes::Helm::NAMESPACE
+      end
+
+      def service_name
+        'prometheus-prometheus-server'
+      end
+
+      def service_port
+        80
+      end
+
       def chart_values_file
         "#{Rails.root}/vendor/#{name}/values.yaml"
       end

app/models/environment.rb

@@ -163,6 +163,18 @@ class Environment < ActiveRecord::Base
     end
   end
 
+  def enabled_clusters
+    slug = self.slug
+    result = project.clusters.enabled.select do |cluster|
+      scope = cluster.environment_scope || '*'
+      File.fnmatch(scope, slug)
+    end
+
+    # sort results by descending order based on environment_scope being longer
+    # thus more closely matching environment slug
+    result.sort_by { |cluster| cluster.environment_scope.length }.reverse!
+  end
+
   def slug
     super.presence || generate_slug
   end

app/models/project_services/prometheus_service.rb

@@ -54,12 +54,16 @@ class PrometheusService < MonitoringService
     { success: false, result: err }
   end
 
+  def with_reactive_cache(cl, *args)
+    yield calculate_reactive_cache(cl, *args)
+  end
+
   def environment_metrics(environment)
     with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics))
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics))
     metrics&.merge(deployment_time: deployment.created_at.to_i) || {}
   end
 
@@ -68,18 +72,24 @@ class PrometheusService < MonitoringService
   end
 
   def additional_deployment_metrics(deployment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.id, &:itself)
+    with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.environment.id, deployment.id, &:itself)
   end
 
   def matched_metrics
-    with_reactive_cache(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself)
+    with_reactive_cache(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, nil, &:itself)
+  end
+
+  def manual_mode?
+    false
   end
 
   # Cache metrics for specific environment
-  def calculate_reactive_cache(query_class_name, *args)
+  def calculate_reactive_cache(query_class_name, environment_id, *args)
     return unless active? && project && !project.pending_delete?
+    client = client_for_environment(environment_id)
 
-    data = Kernel.const_get(query_class_name).new(client).query(*args)
+
+    data = Kernel.const_get(query_class_name).new(client).query(environment_id, *args)
     {
       success: true,
       data: data,
@@ -89,12 +99,51 @@ class PrometheusService < MonitoringService
     { success: false, result: err.message }
   end
 
-  def client
-    @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url)
+  def client(environment_id)
+    if manual_mode?
+      Gitlab::PrometheusClient.new(api_url: api_url)
+    else
+      cluster(environment_id)
+    end
+  end
+
+  def find_cluster_with_prometheus(environment_id)
+    clusters = if environment_id
+                 ::Environment.find_by(id: environment_id).try(:enabled_clusters) || []
+               else
+                 project.clusters.enabled.select { |c| c.environment_scope == '*' || c.environment_scope == '' }
+               end
+
+    clusters.detect { |cluster| cluster.application_prometheus.installed? }
   end
 
   private
 
+  def client_for_environment(environment_id)
+    cluster = find_cluster_with_prometheus(environment_id)
+    return unless cluster
+
+    prometheus = cluster.application_prometheus
+
+    client_through_kube_proxy(cluster.kubeclient,
+                              'service',
+                              prometheus.service_name,
+                              prometheus.service_port,
+                              prometheus.namespace) if cluster.kubeclient
+  end
+
+  def client_through_kube_proxy(kube_client, kind, name, port, namespace = '')
+    rest_client = kube_client.rest_client
+    base_url = rest_client.url
+    proxy_url = kube_client.proxy_url(kind, name, port, namespace)
+
+    Rails.logger.warn rest_client[proxy_url.sub(base_url, '')]
+    Rails.logger.warn proxy_url.sub(base_url, '')
+
+    Gitlab::PrometheusClient.new(api_url: api_url, rest_client: rest_client[proxy_url.sub(base_url, '')], headers: kube_client.headers)
+  end
+
+
   def rename_data_to_metrics(metrics)
     metrics[:metrics] = metrics.delete :data
     metrics

lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb

@@ -4,7 +4,7 @@ module Gitlab
       class AdditionalMetricsDeploymentQuery < BaseQuery
         include QueryAdditionalMetrics
 
-        def query(deployment_id)
+        def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
               common_query_context(

lib/gitlab/prometheus/queries/deployment_query.rb

@@ -2,7 +2,7 @@ module Gitlab
   module Prometheus
     module Queries
       class DeploymentQuery < BaseQuery
-        def query(deployment_id)
+        def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             environment_slug = deployment.environment.slug
 

lib/gitlab/prometheus_client.rb

@@ -3,10 +3,12 @@ module Gitlab
 
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
-    attr_reader :api_url
+    attr_reader :api_url, :rest_client, :headers
 
-    def initialize(api_url:)
+    def initialize(api_url:, rest_client: nil, headers: nil)
       @api_url = api_url
+      @rest_client = rest_client || RestClient::Resource.new(api_url)
+      @headers = headers || {}
     end
 
     def ping
@@ -40,24 +42,15 @@ module Gitlab
     private
 
     def json_api_get(type, args = {})
-      get(join_api_url(type, args))
+      path = ['api', 'v1', type].join('/')
+      get(path, args)
     rescue Errno::ECONNREFUSED
       raise PrometheusError, 'Connection refused'
     end
 
-    def join_api_url(type, args = {})
-      url = URI.parse(api_url)
-    rescue URI::Error
-      raise PrometheusError, "Invalid API URL: #{api_url}"
-    else
-      url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/')
-      url.query = args.to_query
-
-      url.to_s
-    end
-
-    def get(url)
-      handle_response(HTTParty.get(url))
+    def get(path, args)
+      response = rest_client[path].get(headers.merge(params: args))
+      handle_response(response)
     rescue SocketError
       raise PrometheusError, "Can't connect to #{url}"
     rescue OpenSSL::SSL::SSLError
@@ -67,15 +60,20 @@ module Gitlab
     end
 
     def handle_response(response)
-      if response.code == 200 && response['status'] == 'success'
-        response['data'] || {}
+      json_data = json_response(response)
+      if response.code == 200 && json_data['status'] == 'success'
+        json_data['data'] || {}
       elsif response.code == 400
-        raise PrometheusError, response['error'] || 'Bad data received'
+        raise PrometheusError, json_data['error'] || 'Bad data received'
       else
         raise PrometheusError, "#{response.code} - #{response.body}"
       end
     end
 
+    def json_response(response)
+      JSON.parse(response.body)
+    end
+
     def get_result(expected_type)
       data = yield
       data['result'] if data['resultType'] == expected_type