Merge branch '22864-kubernetes-service' into 'master'

Introduce deployment services, starting with a KubernetesService

## What does this MR do?

Adds a minimal `KubernetesService` and introduces the idea of deployment services, generally.

## Are there points in the code the reviewer needs to double check?

## Why was this MR needed?

Two issues scheduled for 8.15 both require a Kubernetes service - #22864 and #23580 - but they use it for very different things. Add a minimal kubernetes service to avoid conflicts later.

## Screenshots

![Screen_Shot_2016-12-14_at_12.44.13](/uploads/77d38baf2a196118f0cf7e2b996a65f3/Screen_Shot_2016-12-14_at_12.44.13.png)

## Does this MR meet the acceptance criteria?

- [x] [Changelog entry](https://docs.gitlab.com/ce/development/changelog.html) added
- [x] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [x] API support added
- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing
- [x] Conform by the [merge request performance guides](http://docs.gitlab.com/ce/development/merge_request_performance_guidelines.html)
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if it does - rebase it please)
- [x] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)

## What are the relevant issue numbers?

Closes #22985

See merge request !7994

Gemfile

@@ -178,6 +178,9 @@ gem 'asana', '~> 0.4.0'
 # FogBugz integration
 gem 'ruby-fogbugz', '~> 0.2.1'
 
+# Kubernetes integration
+gem 'kubeclient', '~> 2.2.0'
+
 # d3
 gem 'd3_rails', '~> 3.5.0'
 

Gemfile.lock

@@ -161,6 +161,8 @@ GEM
     diff-lcs (1.2.5)
     diffy (3.1.0)
     docile (1.1.5)
+    domain_name (0.5.20161021)
+      unf (>= 0.0.5, < 1.0.0)
     doorkeeper (4.2.0)
       railties (>= 4.2)
     dropzonejs-rails (0.7.2)
@@ -318,6 +320,15 @@ GEM
     html2text (0.2.0)
       nokogiri (~> 1.6)
     htmlentities (4.3.4)
+    http (0.9.8)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.1)
+    http_parser.rb (0.6.0)
     httparty (0.13.7)
       json (~> 1.8)
       multi_xml (>= 0.5.2)
@@ -352,6 +363,10 @@ GEM
     knapsack (1.11.0)
       rake
       timecop (>= 0.1.0)
+    kubeclient (2.2.0)
+      http (= 0.9.8)
+      recursive-open-struct (= 1.0.0)
+      rest-client
     launchy (2.4.3)
       addressable (~> 2.3)
     letter_opener (1.4.1)
@@ -388,6 +403,7 @@ GEM
     mysql2 (0.3.20)
     net-ldap (0.12.1)
     net-ssh (3.0.1)
+    netrc (0.11.0)
     newrelic_rpm (3.16.0.318)
     nokogiri (1.6.8)
       mini_portile2 (~> 2.1.0)
@@ -543,6 +559,7 @@ GEM
       json (~> 1.4)
     recaptcha (3.0.0)
       json
+    recursive-open-struct (1.0.0)
     redcarpet (3.3.3)
     redis (3.2.2)
     redis-actionpack (5.0.1)
@@ -568,6 +585,10 @@ GEM
       listen (~> 3.0)
     responders (2.3.0)
       railties (>= 4.2.0, < 5.1)
+    rest-client (2.0.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     rinku (2.0.0)
     rotp (2.1.2)
     rouge (2.0.7)
@@ -859,6 +880,7 @@ DEPENDENCIES
   jwt
   kaminari (~> 0.17.0)
   knapsack (~> 1.11.0)
+  kubeclient (~> 2.2.0)
   letter_opener_web (~> 1.3.0)
   license_finder (~> 2.1.0)
   licensee (~> 8.0.0)

app/controllers/concerns/service_params.rb

@@ -18,7 +18,7 @@ module ServiceParams
                     :add_pusher, :send_from_committer_email, :disable_diffs,
                     :external_wiki_url, :notify, :color,
                     :server_host, :server_port, :default_irc_uri, :enable_ssl_verification,
-                    :jira_issue_transition_id, :url, :project_key]
+                    :jira_issue_transition_id, :url, :project_key, :ca_pem, :namespace]
 
   # Parameters to ignore if no value is specified
   FILTER_BLANK_PARAMS = [:password]

app/models/project.rb

@@ -106,6 +106,7 @@ class Project < ActiveRecord::Base
   has_one :bugzilla_service, dependent: :destroy
   has_one :gitlab_issue_tracker_service, dependent: :destroy, inverse_of: :project
   has_one :external_wiki_service, dependent: :destroy
+  has_one :kubernetes_service, dependent: :destroy, inverse_of: :project
 
   has_one  :forked_project_link,  dependent: :destroy, foreign_key: "forked_to_project_id"
   has_one  :forked_from_project,  through:   :forked_project_link
@@ -742,6 +743,14 @@ class Project < ActiveRecord::Base
     @ci_service ||= ci_services.reorder(nil).find_by(active: true)
   end
 
+  def deployment_services
+    services.where(category: :deployment)
+  end
+
+  def deployment_service
+    @deployment_service ||= deployment_services.reorder(nil).find_by(active: true)
+  end
+
   def jira_tracker?
     issues_tracker.to_param == 'jira'
   end

app/models/project_services/deployment_service.rb

@@ -0,0 +1,11 @@
+# Base class for deployment services
+#
+# These services integrate with a deployment solution like Kubernetes/OpenShift,
+# Mesosphere, etc, to provide additional features to environments.
+class DeploymentService < Service
+  default_value_for :category, 'deployment'
+
+  def supported_events
+    []
+  end
+end

app/models/project_services/kubernetes_service.rb

@@ -0,0 +1,118 @@
+class KubernetesService < DeploymentService
+  # Namespace defaults to the project path, but can be overridden in case that
+  # is an invalid or inappropriate name
+  prop_accessor :namespace
+
+  #  Access to kubernetes is directly through the API
+  prop_accessor :api_url
+
+  # Bearer authentication
+  # TODO:  user/password auth, client certificates
+  prop_accessor :token
+
+  # Provide a custom CA bundle for self-signed deployments
+  prop_accessor :ca_pem
+
+  with_options presence: true, if: :activated? do
+    validates :api_url, url: true
+    validates :token
+
+    validates :namespace,
+      format: {
+        with: Gitlab::Regex.kubernetes_namespace_regex,
+        message: Gitlab::Regex.kubernetes_namespace_regex_message,
+      },
+      length: 1..63
+  end
+
+  def initialize_properties
+    if properties.nil?
+      self.properties = {}
+      self.namespace = project.path if project.present?
+    end
+  end
+
+  def title
+    'Kubernetes'
+  end
+
+  def description
+    'Kubernetes / Openshift integration'
+  end
+
+  def help
+    ''
+  end
+
+  def to_param
+    'kubernetes'
+  end
+
+  def fields
+    [
+        { type: 'text',
+          name: 'namespace',
+          title: 'Kubernetes namespace',
+          placeholder: 'Kubernetes namespace',
+        },
+        { type: 'text',
+          name: 'api_url',
+          title: 'API URL',
+          placeholder: 'Kubernetes API URL, like https://kube.example.com/',
+        },
+        { type: 'text',
+          name: 'token',
+          title: 'Service token',
+          placeholder: 'Service token',
+        },
+        { type: 'textarea',
+          name: 'ca_pem',
+          title: 'Custom CA bundle',
+          placeholder: 'Certificate Authority bundle (PEM format)',
+        },
+    ]
+  end
+
+  # Check we can connect to the Kubernetes API
+  def test(*args)
+    kubeclient = build_kubeclient
+    kubeclient.discover
+
+    { success: kubeclient.discovered, result: "Checked API discovery endpoint" }
+  rescue => err
+    { success: false, result: err }
+  end
+
+  private
+
+  def build_kubeclient(api_path = '/api', api_version = 'v1')
+    return nil unless api_url && namespace && token
+
+    url = URI.parse(api_url)
+    url.path = url.path[0..-2] if url.path[-1] == "/"
+    url.path += api_path
+
+    ::Kubeclient::Client.new(
+      url,
+      api_version,
+      ssl_options: kubeclient_ssl_options,
+      auth_options: kubeclient_auth_options,
+      http_proxy_uri: ENV['http_proxy']
+    )
+  end
+
+  def kubeclient_ssl_options
+    opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+    if ca_pem.present?
+      opts[:cert_store] = OpenSSL::X509::Store.new
+      opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+    end
+
+    opts
+  end
+
+  def kubeclient_auth_options
+    { bearer_token: token }
+  end
+end

app/models/service.rb

@@ -214,6 +214,7 @@ class Service < ActiveRecord::Base
       hipchat
       irker
       jira
+      kubernetes
       mattermost_slash_commands
       pipelines_email
       pivotaltracker

changelogs/unreleased/22864-kubernetes-service.yml

@@ -0,0 +1,4 @@
+---
+title: Introduce deployment services, starting with a KubernetesService
+merge_request: 7994
+author: 

doc/project_services/kubernetes.md

@@ -0,0 +1,38 @@
+# GitLab Kubernetes / OpenShift integration
+
+GitLab can be configured to interact with Kubernetes, or other systems using the
+Kubernetes API (such as OpenShift).
+
+Each project can be configured to connect to a different Kubernetes cluster, see
+the [configuration](#configuration) section.
+
+If you have a single cluster that you want to use for all your projects,
+you can pre-fill the settings page with a default template. To configure the
+template, see the [Services Templates](services-templates.md) document.
+
+## Configuration
+
+![Kubernetes configuration settings](img/kubernetes_configuration.png)
+
+The Kubernetes service takes the following arguments:
+
+1. Kubernetes namespace
+1. API URL
+1. Service token
+1. Custom CA bundle
+
+The API URL is the URL that GitLab uses to access the Kubernetes API. Kubernetes
+exposes several APIs - we want the "base" URL that is common to all of them,
+e.g., `https://kubernetes.example.com` rather than `https://kubernetes.example.com/api/v1`.
+
+GitLab authenticates against Kubernetes using service tokens, which are
+scoped to a particular `namespace`. If you don't have a service token yet,
+you can follow the
+[Kubernetes documentation](http://kubernetes.io/docs/user-guide/service-accounts/)
+to create one. You can also view or create service tokens in the
+[Kubernetes dashboard](http://kubernetes.io/docs/user-guide/ui/) - visit
+`Config -> Secrets`.
+
+Fill in the service token and namespace according to the values you just got.
+If the API is using a self-signed TLS certificate, you'll also need to include
+the `ca.crt` contents as the `Custom CA bundle`.

doc/project_services/project_services.md

@@ -42,6 +42,7 @@ further configuration instructions and details. Contributions are welcome.
 | [Irker (IRC gateway)](irker.md) | Send IRC messages, on update, to a list of recipients through an Irker gateway |
 | [JIRA](jira.md) | JIRA issue tracker |
 | JetBrains TeamCity CI | A continuous integration and build server |
+| [Kubernetes](kubernetes.md) | A containerized deployment service |
 | [Mattermost slash commands](mattermost_slash_commands.md) | Mattermost chat and ChatOps slash commands |
 | PivotalTracker | Project Management Software (Source Commits Endpoint) |
 | Pushover | Pushover makes it easy to get real-time notifications on your Android device, iPhone, iPad, and Desktop |

lib/api/services.rb

@@ -351,6 +351,34 @@ module API
           desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
         }
       ],
+
+      'kubernetes' => [
+        {
+          required: true,
+          name: :namespace,
+          type: String,
+          desc: 'The Kubernetes namespace to use'
+        },
+        {
+          required: true,
+          name: :api_url,
+          type: String,
+          desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com'
+        },
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The service token to authenticate against the Kubernetes cluster with'
+        },
+        {
+          required: false,
+          name: :ca_pem,
+          type: String,
+          desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)'
+        },
+      ],
+
       'mattermost-slash-commands' => [
         {
           required: true,

lib/gitlab/regex.rb

@@ -123,5 +123,13 @@ module Gitlab
     def environment_name_regex_message
       "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces"
     end
+
+    def kubernetes_namespace_regex
+      /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
+    end
+
+    def kubernetes_namespace_regex_message
+      "can contain only letters, digits or '-', and cannot start or end with '-'"
+    end
   end
 end

spec/factories/projects.rb

@@ -133,4 +133,17 @@ FactoryGirl.define do
       )
     end
   end
+
+  factory :kubernetes_project, parent: :empty_project do
+    after :create do |project|
+      project.create_kubernetes_service(
+        active: true,
+        properties: {
+          namespace: project.path,
+          api_url: 'https://kubernetes.example.com/api',
+          token: 'a' * 40,
+        }
+      )
+    end
+  end
 end

spec/lib/gitlab/import_export/all_models.yml

@@ -147,6 +147,7 @@ project:
 - bugzilla_service
 - gitlab_issue_tracker_service
 - external_wiki_service
+- kubernetes_service
 - forked_project_link
 - forked_from_project
 - forked_project_links

spec/models/project_services/kubernetes_service_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+
+describe KubernetesService, models: true do
+  let(:project) { create(:empty_project) }
+
+  describe "Associations" do
+    it { is_expected.to belong_to :project }
+  end
+
+  describe 'Validations' do
+    context 'when service is active' do
+      before { subject.active = true }
+      it { is_expected.to validate_presence_of(:namespace) }
+      it { is_expected.to validate_presence_of(:api_url) }
+      it { is_expected.to validate_presence_of(:token) }
+
+      context 'namespace format' do
+        before do
+          subject.project = project
+          subject.api_url = "http://example.com"
+          subject.token = "test"
+        end
+
+        {
+          'foo'  => true,
+          '1foo' => true,
+          'foo1' => true,
+          'foo-bar' => true,
+          '-foo' => false,
+          'foo-' => false,
+          'a' * 63 => true,
+          'a' * 64 => false,
+          'a.b' => false,
+          'a*b' => false,
+        }.each do |namespace, validity|
+          it "should validate #{namespace} as #{validity ? 'valid' : 'invalid'}" do
+            subject.namespace = namespace
+
+            expect(subject.valid?).to eq(validity)
+          end
+        end
+      end
+    end
+
+    context 'when service is inactive' do
+      before { subject.active = false }
+      it { is_expected.not_to validate_presence_of(:namespace) }
+      it { is_expected.not_to validate_presence_of(:api_url) }
+      it { is_expected.not_to validate_presence_of(:token) }
+    end
+  end
+
+  describe '#initialize_properties' do
+    context 'with a project' do
+      it 'defaults to the project name' do
+        expect(described_class.new(project: project).namespace).to eq(project.name)
+      end
+    end
+
+    context 'without a project' do
+      it 'leaves the namespace unset' do
+        expect(described_class.new.namespace).to be_nil
+      end
+    end
+  end
+
+  describe '#test' do
+    let(:project) { create(:kubernetes_project) }
+    let(:service) { project.kubernetes_service }
+    let(:discovery_url) { service.api_url + '/api/v1' }
+
+    # JSON response body from Kubernetes GET /api/v1 request
+    let(:discovery_response) { { "kind" => "APIResourceList", "groupVersion" => "v1", "resources" => [] }.to_json }
+
+    context 'with path prefix in api_url' do
+      let(:discovery_url) { 'https://kubernetes.example.com/prefix/api/v1' }
+
+      before do
+        service.api_url = 'https://kubernetes.example.com/prefix/'
+      end
+
+      it 'tests with the prefix' do
+        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
+
+        expect(service.test[:success]).to be_truthy
+        expect(WebMock).to have_requested(:get, discovery_url).once
+      end
+    end
+
+    context 'with custom CA certificate' do
+      let(:certificate) { "CA PEM DATA" }
+      before do
+        service.update_attributes!(ca_pem: certificate)
+      end
+
+      it 'is added to the certificate store' do
+        cert = double("certificate")
+
+        expect(OpenSSL::X509::Certificate).to receive(:new).with(certificate).and_return(cert)
+        expect_any_instance_of(OpenSSL::X509::Store).to receive(:add_cert).with(cert)
+        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
+
+        expect(service.test[:success]).to be_truthy
+        expect(WebMock).to have_requested(:get, discovery_url).once
+      end
+    end
+
+    context 'success' do
+      it 'reads the discovery endpoint' do
+        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
+
+        expect(service.test[:success]).to be_truthy
+        expect(WebMock).to have_requested(:get, discovery_url).once
+      end
+    end
+
+    context 'failure' do
+      it 'fails to read the discovery endpoint' do
+        WebMock.stub_request(:get, discovery_url).to_return(status: 404)
+
+        expect(service.test[:success]).to be_falsy
+        expect(WebMock).to have_requested(:get, discovery_url).once
+      end
+    end
+  end
+end