Move migration file to post-migration. Use EachBatch. batch_size 1

db/migrate/20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb

@@ -1,90 +0,0 @@
-class MigrateGcpClustersToNewClustersArchitectures < ActiveRecord::Migration
-  DOWNTIME = false
-
-  def up
-    gcp_clusters = ActiveRecord::Base.connection.select_all('SELECT * from gcp_clusters;')
-
-    rows_for_clusters = Array.new
-    rows_for_cluster_projects = Array.new
-    rows_for_cluster_providers_gcp = Array.new
-    rows_for_cluster_platforms_kubernetes = Array.new
-
-    gcp_clusters.each do |gcp_cluster|
-      rows_for_clusters << params_for_clusters(gcp_cluster)
-      rows_for_cluster_projects << params_for_cluster_projects(gcp_cluster)
-      rows_for_cluster_providers_gcp << params_for_cluster_providers_gcp(gcp_cluster)
-      rows_for_cluster_platforms_kubernetes << params_for_cluster_platforms_kubernetes(gcp_cluster)
-    end
-
-    Gitlab::Database.bulk_insert('clusters', rows_for_clusters)
-    Gitlab::Database.bulk_insert('cluster_projects', rows_for_cluster_projects)
-    Gitlab::Database.bulk_insert('cluster_providers_gcp', rows_for_cluster_providers_gcp)
-    Gitlab::Database.bulk_insert('cluster_platforms_kubernetes', rows_for_cluster_platforms_kubernetes)
-  end
-
-  def down
-    execute('DELETE FROM clusters')
-  end
-
-  private
-
-  def params_for_clusters(gcp_cluster)
-    {
-      id: gcp_cluster['id'],
-      user_id: gcp_cluster['user_id'],
-      enabled: gcp_cluster['enabled'],
-      name: gcp_cluster['gcp_cluster_name'],
-      provider_type: Clusters::Cluster.provider_types[:gcp],
-      platform_type: Clusters::Cluster.platform_types[:kubernetes],
-      created_at: gcp_cluster['created_at'],
-      updated_at: gcp_cluster['updated_at']
-    }
-  end
-
-  def params_for_cluster_projects(gcp_cluster)
-    {
-      cluster_id: gcp_cluster['id'],
-      project_id: gcp_cluster['project_id'],
-      created_at: gcp_cluster['created_at'],
-      updated_at: gcp_cluster['updated_at']
-    }
-  end
-
-  def params_for_cluster_providers_gcp(gcp_cluster)
-    {
-      cluster_id: gcp_cluster['id'],
-      status: gcp_cluster['status'],
-      status_reason: gcp_cluster['status_reason'],
-      gcp_project_id: gcp_cluster['gcp_project_id'],
-      zone: gcp_cluster['gcp_cluster_zone'],
-      num_nodes: gcp_cluster['gcp_cluster_size'],
-      machine_type: gcp_cluster['gcp_machine_type'],
-      operation_id: gcp_cluster['gcp_operation_id'],
-      endpoint: gcp_cluster['endpoint'],
-      encrypted_access_token: gcp_cluster['encrypted_gcp_token'],
-      encrypted_access_token_iv: gcp_cluster['encrypted_gcp_token_iv'],
-      created_at: gcp_cluster['created_at'],
-      updated_at: gcp_cluster['updated_at']
-    }
-  end
-
-  def params_for_cluster_platforms_kubernetes(gcp_cluster)
-    {
-      cluster_id: gcp_cluster['id'],
-      api_url: api_url(gcp_cluster['endpoint']),
-      ca_cert: gcp_cluster['ca_cert'],
-      namespace: gcp_cluster['project_namespace'],
-      username: gcp_cluster['username'],
-      encrypted_password: gcp_cluster['encrypted_password'],
-      encrypted_password_iv: gcp_cluster['encrypted_password_iv'],
-      encrypted_token: gcp_cluster['encrypted_kubernetes_token'],
-      encrypted_token_iv: gcp_cluster['encrypted_kubernetes_token_iv'],
-      created_at: gcp_cluster['created_at'],
-      updated_at: gcp_cluster['updated_at']
-    }
-  end
-
-  def api_url(endpoint)
-    endpoint ? 'https://' + endpoint : nil
-  end
-end

db/post_migrate/20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb

@@ -0,0 +1,99 @@
+class MigrateGcpClustersToNewClustersArchitectures < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class GcpCluster < ActiveRecord::Base
+    self.table_name = 'gcp_clusters'
+
+    belongs_to :project, class_name: 'Project'
+
+    include EachBatch
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'Project'
+    has_one :provider_gcp, class_name: 'ProvidersGcp'
+    has_one :platform_kubernetes, class_name: 'PlatformsKubernetes'
+
+    accepts_nested_attributes_for :provider_gcp
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_one :cluster_project, class_name: 'ClustersProject'
+    has_one :cluster, through: :cluster_project, class_name: 'Cluster'
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'Cluster'
+    belongs_to :project, class_name: 'Project'
+  end
+
+  class ProvidersGcp < ActiveRecord::Base
+    self.table_name = 'cluster_providers_gcp'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+  end
+
+  def up
+    GcpCluster.all.find_each(batch_size: 1) do |gcp_cluster|
+      Cluster.create(
+        enabled: gcp_cluster.enabled,
+        user_id: gcp_cluster.user_id,
+        name: gcp_cluster.gcp_cluster_name,
+        provider_type: Cluster.provider_types[:gcp],
+        platform_type: Cluster.platform_types[:kubernetes],
+        projects: [gcp_cluster.project],
+        provider_gcp_attributes: {
+          status: gcp_cluster.status,
+          status_reason: gcp_cluster.status_reason,
+          gcp_project_id: gcp_cluster.gcp_project_id,
+          zone: gcp_cluster.gcp_cluster_zone,
+          num_nodes: gcp_cluster.gcp_cluster_size,
+          machine_type: gcp_cluster.gcp_machine_type,
+          operation_id: gcp_cluster.gcp_operation_id,
+          endpoint: gcp_cluster.endpoint,
+          encrypted_access_token: gcp_cluster.encrypted_gcp_token,
+          encrypted_access_token_iv: gcp_cluster.encrypted_gcp_token_iv,
+        },
+        platform_kubernetes_attributes: {
+          cluster_id: gcp_cluster.id,
+          api_url: api_url(gcp_cluster.endpoint),
+          ca_cert: gcp_cluster.ca_cert,
+          namespace: gcp_cluster.project_namespace,
+          username: gcp_cluster.username,
+          encrypted_password: gcp_cluster.encrypted_password,
+          encrypted_password_iv: gcp_cluster.encrypted_password_iv,
+          encrypted_token: gcp_cluster.encrypted_kubernetes_token,
+          encrypted_token_iv: gcp_cluster.encrypted_kubernetes_token_iv
+        } )
+    end
+  end
+
+  def down
+    execute('DELETE FROM clusters')
+  end
+
+  private
+
+  def api_url(endpoint)
+    endpoint ? 'https://' + endpoint : nil
+  end
+end

spec/migrations/migrate_gcp_clusters_to_new_clusters_architectures_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require Rails.root.join('db', 'migrate', '20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb')
+require Rails.root.join('db', 'post_migrate', '20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb')
 
 describe MigrateGcpClustersToNewClustersArchitectures, :migration do
   let(:project) { create(:project) }
@@ -55,10 +55,9 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.name).to eq(gcp_cluster_name.delete!("'"))
       expect(cluster.provider_type).to eq('gcp')
       expect(cluster.platform_type).to eq('kubernetes')
-      expect(cluster.created_at).to eq(created_at)
-      expect(cluster.updated_at).to eq(updated_at)
 
       expect(cluster.project).to eq(project)
+      expect(project.cluster).to eq(cluster)
 
       expect(cluster.provider_gcp.cluster).to eq(cluster)
       expect(cluster.provider_gcp.status).to eq(status)
@@ -71,8 +70,6 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.provider_gcp.endpoint).to be_nil
       expect(cluster.provider_gcp.encrypted_access_token).to eq(tr(encrypted_gcp_token))
       expect(cluster.provider_gcp.encrypted_access_token_iv).to eq(tr(encrypted_gcp_token_iv))
-      expect(cluster.provider_gcp.created_at).to eq(created_at)
-      expect(cluster.provider_gcp.updated_at).to eq(updated_at)
 
       expect(cluster.platform_kubernetes.cluster).to eq(cluster)
       expect(cluster.platform_kubernetes.api_url).to be_nil
@@ -83,8 +80,6 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.platform_kubernetes.encrypted_password_iv).to be_nil
       expect(cluster.platform_kubernetes.encrypted_token).to be_nil
       expect(cluster.platform_kubernetes.encrypted_token_iv).to be_nil
-      expect(cluster.platform_kubernetes.created_at).to eq(created_at)
-      expect(cluster.platform_kubernetes.updated_at).to eq(updated_at)
     end
   end
 
@@ -137,10 +132,9 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.name).to eq(tr(gcp_cluster_name))
       expect(cluster.provider_type).to eq('gcp')
       expect(cluster.platform_type).to eq('kubernetes')
-      expect(cluster.created_at).to eq(created_at)
-      expect(cluster.updated_at).to eq(updated_at)
 
       expect(cluster.project).to eq(project)
+      expect(project.cluster).to eq(cluster)
 
       expect(cluster.provider_gcp.cluster).to eq(cluster)
       expect(cluster.provider_gcp.status).to eq(status)
@@ -153,8 +147,6 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.provider_gcp.endpoint).to eq(tr(endpoint))
       expect(cluster.provider_gcp.encrypted_access_token).to eq(tr(encrypted_gcp_token))
       expect(cluster.provider_gcp.encrypted_access_token_iv).to eq(tr(encrypted_gcp_token_iv))
-      expect(cluster.provider_gcp.created_at).to eq(created_at)
-      expect(cluster.provider_gcp.updated_at).to eq(updated_at)
 
       expect(cluster.platform_kubernetes.cluster).to eq(cluster)
       expect(cluster.platform_kubernetes.api_url).to eq('https://' + tr(endpoint))
@@ -165,8 +157,6 @@ describe MigrateGcpClustersToNewClustersArchitectures, :migration do
       expect(cluster.platform_kubernetes.encrypted_password_iv).to eq(tr(encrypted_password_iv))
       expect(cluster.platform_kubernetes.encrypted_token).to eq(tr(encrypted_kubernetes_token))
       expect(cluster.platform_kubernetes.encrypted_token_iv).to eq(tr(encrypted_kubernetes_token_iv))
-      expect(cluster.platform_kubernetes.created_at).to eq(created_at)
-      expect(cluster.platform_kubernetes.updated_at).to eq(updated_at)
     end
   end