Add background migration for Kubernetes Namespaces

changelogs/unreleased/51716-add-kubernetes-namespace-background-migration.yml

@@ -0,0 +1,5 @@
+---
+title: Add background migration to populate Kubernetes namespaces
+merge_request: 22433
+author:
+type: added

db/post_migrate/20181022173835_enqueue_populate_cluster_kubernetes_namespace.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class EnqueuePopulateClusterKubernetesNamespace < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'PopulateClusterKubernetesNamespaceTable'.freeze
+
+  disable_ddl_transaction!
+
+  def up
+    BackgroundMigrationWorker.perform_async(MIGRATION)
+  end
+
+  def down
+    Clusters::KubernetesNamespace.delete_all
+  end
+end

lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+#
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateClusterKubernetesNamespaceTable
+      include Gitlab::Database::MigrationHelpers
+
+      BATCH_SIZE = 1_000
+
+      module Migratable
+        class KubernetesNamespace < ActiveRecord::Base
+          self.table_name = 'clusters_kubernetes_namespaces'
+        end
+
+        class ClusterProject < ActiveRecord::Base
+          include EachBatch
+
+          self.table_name = 'cluster_projects'
+
+          belongs_to :project
+
+          def self.with_no_kubernetes_namespace
+            where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id))
+          end
+
+          def namespace
+            slug = "#{project.path}-#{project.id}".downcase
+            slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
+          end
+
+          def service_account
+            "#{namespace}-service-account"
+          end
+        end
+
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+        end
+      end
+
+      def perform
+        cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index|
+          sql_values = sql_values_for(cluster_projects_batch)
+
+          insert_into_cluster_kubernetes_namespace(sql_values)
+        end
+      end
+
+      private
+
+      def cluster_projects_with_no_kubernetes_namespace
+        Migratable::ClusterProject.with_no_kubernetes_namespace
+      end
+
+      def sql_values_for(cluster_projects)
+        cluster_projects.map do |cluster_project|
+          values_for_cluster_project(cluster_project)
+        end
+      end
+
+      def values_for_cluster_project(cluster_project)
+        {
+          cluster_project_id: cluster_project.id,
+          cluster_id: cluster_project.cluster_id,
+          project_id: cluster_project.project_id,
+          namespace: cluster_project.namespace,
+          service_account_name: cluster_project.service_account,
+          created_at: 'NOW()',
+          updated_at: 'NOW()'
+        }
+      end
+
+      def insert_into_cluster_kubernetes_namespace(rows)
+        Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name,
+                                     rows,
+                                     disable_quote: [:created_at, :updated_at])
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable, :migration, schema: 20181022173835 do
+  let(:migration) { described_class.new }
+  let(:clusters) { create_list(:cluster, 10, :project, :provided_by_gcp) }
+
+  before do
+    clusters
+  end
+
+  shared_examples 'consistent kubernetes namespace attributes' do
+    it 'should populate namespace and service account information' do
+      subject
+
+      clusters_with_namespace.each do |cluster|
+        project = cluster.project
+        cluster_project = cluster.cluster_projects.first
+        namespace = "#{project.path}-#{project.id}"
+        kubernetes_namespace = cluster.reload.kubernetes_namespace
+
+        expect(kubernetes_namespace).to be_present
+        expect(kubernetes_namespace.cluster_project).to eq(cluster_project)
+        expect(kubernetes_namespace.project).to eq(cluster_project.project)
+        expect(kubernetes_namespace.cluster).to eq(cluster_project.cluster)
+        expect(kubernetes_namespace.namespace).to eq(namespace)
+        expect(kubernetes_namespace.service_account_name).to eq("#{namespace}-service-account")
+      end
+    end
+  end
+
+  subject { migration.perform }
+
+  context 'when no Clusters::Project has a Clusters::KubernetesNamespace' do
+    let(:cluster_projects) { Clusters::Project.all }
+
+    it 'should create a Clusters::KubernetesNamespace per Clusters::Project' do
+      expect do
+        subject
+      end.to change(Clusters::KubernetesNamespace, :count).by(cluster_projects.count)
+    end
+
+    it_behaves_like 'consistent kubernetes namespace attributes' do
+      let(:clusters_with_namespace) { clusters }
+    end
+  end
+
+  context 'when every Clusters::Project has Clusters::KubernetesNamespace' do
+    before do
+      clusters.each do |cluster|
+        create(:cluster_kubernetes_namespace,
+               cluster_project: cluster.cluster_projects.first,
+               cluster: cluster,
+               project: cluster.project)
+      end
+    end
+
+    it 'should not create any Clusters::KubernetesNamespace' do
+      expect do
+        subject
+      end.not_to change(Clusters::KubernetesNamespace, :count)
+    end
+  end
+
+  context 'when only some Clusters::Project have Clusters::KubernetesNamespace related' do
+    let(:with_kubernetes_namespace) { clusters.first(6) }
+    let(:with_no_kubernetes_namespace) { clusters.last(4) }
+
+    before do
+      with_kubernetes_namespace.each do |cluster|
+        create(:cluster_kubernetes_namespace,
+               cluster_project: cluster.cluster_projects.first,
+               cluster: cluster,
+               project: cluster.project)
+      end
+    end
+
+    it 'creates limited number of Clusters::KubernetesNamespace' do
+      expect do
+        subject
+      end.to change(Clusters::KubernetesNamespace, :count).by(with_no_kubernetes_namespace.count)
+    end
+
+    it 'should not modify clusters with Clusters::KubernetesNamespace' do
+      subject
+
+      with_kubernetes_namespace.each do |cluster|
+        expect(cluster.kubernetes_namespaces.count).to eq(1)
+      end
+    end
+
+    it_behaves_like 'consistent kubernetes namespace attributes' do
+      let(:clusters_with_namespace) { with_no_kubernetes_namespace }
+    end
+  end
+end