Add background migrations to arhive legacy traces

db/post_migrate/20180529152628_archive_legacy_traces.rb

@@ -0,0 +1,44 @@
+class ArchiveLegacyTraces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 10_000
+  BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces'
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+    self.inheritance_column = :_type_disabled # Disable STI
+
+    scope :finished, -> { where(status: [:success, :failed, :canceled]) }
+
+    scope :without_new_traces, ->() do
+      where('NOT EXISTS (?)',
+        ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+    end
+  end
+
+  class JobArtifact < ActiveRecord::Base
+    self.table_name = 'ci_job_artifacts'
+
+    enum file_type: {
+      archive: 1,
+      metadata: 2,
+      trace: 3
+    }
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      ::ArchiveLegacyTraces::Build.finished.without_new_traces,
+      BACKGROUND_MIGRATION_CLASS,
+      5.minutes,
+      batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # noop
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180529093006) do
+ActiveRecord::Schema.define(version: 20180529152628) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

lib/gitlab/background_migration/archive_legacy_traces.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class ArchiveLegacyTraces
+      class Build < ActiveRecord::Base
+        include ::HasStatus
+
+        self.table_name = 'ci_builds'
+        self.inheritance_column = :_type_disabled # Disable STI
+
+        belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project'
+        has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id
+        has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk'
+    
+        scope :finished, -> { where(status: [:success, :failed, :canceled]) }
+
+        scope :without_new_traces, ->() do
+          finished.where('NOT EXISTS (?)',
+            BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+        end
+
+        def trace
+          ::Gitlab::Ci::Trace.new(self)
+        end
+
+        def trace=(data)
+          raise NotImplementedError
+        end
+    
+        def old_trace
+          read_attribute(:trace)
+        end
+    
+        def erase_old_trace!
+          update_column(:trace, nil)
+        end
+      end
+
+      class JobArtifact < ActiveRecord::Base
+        self.table_name = 'ci_job_artifacts'
+
+        belongs_to :build
+        belongs_to :project
+
+        mount_uploader :file, JobArtifactUploader
+
+        enum file_type: {
+          archive: 1,
+          metadata: 2,
+          trace: 3
+        }
+      end
+
+      class BuildTraceChunk < ActiveRecord::Base
+        self.table_name = 'ci_build_trace_chunks'
+
+        belongs_to :build
+      end
+
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+
+        has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build'
+      end
+
+      def perform(start_id, stop_id)
+        BackgroundMigration::ArchiveLegacyTraces::Build
+          .finished
+          .without_new_traces
+          .where(id: (start_id..stop_id)).find_each do |build|
+            build.trace.archive!
+          end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:builds) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+
+  before do
+    namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
+    build = builds.create!(id: 1, project_id: 123, status: 'success')
+
+    @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path,
+      build.created_at.utc.strftime("%Y_%m"),
+      build.project_id.to_s)
+    
+    FileUtils.mkdir_p(@legacy_trace_dir)
+
+    @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log")
+  end
+
+  context 'when trace file exsits at the right place' do
+    before do
+      File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') }
+    end
+
+    it 'correctly archive legacy traces' do
+      expect(job_artifacts.count).to eq(0)
+      expect(File.exist?(@legacy_trace_path)).to be_truthy
+
+      described_class.new.perform(1, 1)
+
+      expect(job_artifacts.count).to eq(1)
+      expect(File.exist?(@legacy_trace_path)).to be_falsy
+      expect(File.read(new_trace_path)).to eq('aiueo')
+    end
+  end
+
+  context 'when trace file does not exsits at the right place' do
+    it 'correctly archive legacy traces' do
+      expect(job_artifacts.count).to eq(0)
+      expect(File.exist?(@legacy_trace_path)).to be_falsy
+
+      described_class.new.perform(1, 1)
+
+      expect(job_artifacts.count).to eq(0)
+    end
+  end
+
+  def new_trace_path
+    job_artifact = job_artifacts.first
+
+    disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s)
+    creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d')
+
+    File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash,
+      creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log')
+  end
+end

spec/migrations/archive_legacy_traces_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces')
+
+describe ArchiveLegacyTraces, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:builds) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+
+  before do
+    namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
+    build = builds.create!(id: 1)
+
+    @legacy_trace_path = File.join(
+      Settings.gitlab_ci.builds_path,
+      build.created_at.utc.strftime("%Y_%m"),
+      build.project_id.to_s,
+      "#{job.id}.log"
+    )
+
+    File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') }
+  end
+
+  it 'correctly archive legacy traces' do
+    expect(job_artifacts.count).to eq(0)
+    expect(File.exist?(@legacy_trace_path)).to be_truthy
+
+    migrate!
+
+    expect(job_artifacts.count).to eq(1)
+    expect(File.exist?(@legacy_trace_path)).to be_falsy
+    expect(File.exist?(new_trace_path)).to be_truthy
+  end
+
+  def new_trace_path
+    job_artifact = job_artifacts.first
+
+    disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s)
+    creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d')
+
+    File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
+      creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s)
+  end
+end