Add background migrations to arhive legacy traces
This commit is contained in:
parent
ee111285ec
commit
89b4304f12
5 changed files with 229 additions and 1 deletions
44
db/post_migrate/20180529152628_archive_legacy_traces.rb
Normal file
44
db/post_migrate/20180529152628_archive_legacy_traces.rb
Normal file
|
@ -0,0 +1,44 @@
|
||||||
|
class ArchiveLegacyTraces < ActiveRecord::Migration
|
||||||
|
include Gitlab::Database::MigrationHelpers
|
||||||
|
|
||||||
|
DOWNTIME = false
|
||||||
|
BATCH_SIZE = 10_000
|
||||||
|
BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces'
|
||||||
|
|
||||||
|
disable_ddl_transaction!
|
||||||
|
|
||||||
|
class Build < ActiveRecord::Base
|
||||||
|
include EachBatch
|
||||||
|
self.table_name = 'ci_builds'
|
||||||
|
self.inheritance_column = :_type_disabled # Disable STI
|
||||||
|
|
||||||
|
scope :finished, -> { where(status: [:success, :failed, :canceled]) }
|
||||||
|
|
||||||
|
scope :without_new_traces, ->() do
|
||||||
|
where('NOT EXISTS (?)',
|
||||||
|
::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
class JobArtifact < ActiveRecord::Base
|
||||||
|
self.table_name = 'ci_job_artifacts'
|
||||||
|
|
||||||
|
enum file_type: {
|
||||||
|
archive: 1,
|
||||||
|
metadata: 2,
|
||||||
|
trace: 3
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def up
|
||||||
|
queue_background_migration_jobs_by_range_at_intervals(
|
||||||
|
::ArchiveLegacyTraces::Build.finished.without_new_traces,
|
||||||
|
BACKGROUND_MIGRATION_CLASS,
|
||||||
|
5.minutes,
|
||||||
|
batch_size: BATCH_SIZE)
|
||||||
|
end
|
||||||
|
|
||||||
|
def down
|
||||||
|
# noop
|
||||||
|
end
|
||||||
|
end
|
|
@ -11,7 +11,7 @@
|
||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema.define(version: 20180529093006) do
|
ActiveRecord::Schema.define(version: 20180529152628) do
|
||||||
|
|
||||||
# These are extensions that must be enabled in order to support this database
|
# These are extensions that must be enabled in order to support this database
|
||||||
enable_extension "plpgsql"
|
enable_extension "plpgsql"
|
||||||
|
|
79
lib/gitlab/background_migration/archive_legacy_traces.rb
Normal file
79
lib/gitlab/background_migration/archive_legacy_traces.rb
Normal file
|
@ -0,0 +1,79 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
# rubocop:disable Metrics/AbcSize
|
||||||
|
# rubocop:disable Style/Documentation
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module BackgroundMigration
|
||||||
|
class ArchiveLegacyTraces
|
||||||
|
class Build < ActiveRecord::Base
|
||||||
|
include ::HasStatus
|
||||||
|
|
||||||
|
self.table_name = 'ci_builds'
|
||||||
|
self.inheritance_column = :_type_disabled # Disable STI
|
||||||
|
|
||||||
|
belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project'
|
||||||
|
has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id
|
||||||
|
has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk'
|
||||||
|
|
||||||
|
scope :finished, -> { where(status: [:success, :failed, :canceled]) }
|
||||||
|
|
||||||
|
scope :without_new_traces, ->() do
|
||||||
|
finished.where('NOT EXISTS (?)',
|
||||||
|
BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
|
||||||
|
end
|
||||||
|
|
||||||
|
def trace
|
||||||
|
::Gitlab::Ci::Trace.new(self)
|
||||||
|
end
|
||||||
|
|
||||||
|
def trace=(data)
|
||||||
|
raise NotImplementedError
|
||||||
|
end
|
||||||
|
|
||||||
|
def old_trace
|
||||||
|
read_attribute(:trace)
|
||||||
|
end
|
||||||
|
|
||||||
|
def erase_old_trace!
|
||||||
|
update_column(:trace, nil)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
class JobArtifact < ActiveRecord::Base
|
||||||
|
self.table_name = 'ci_job_artifacts'
|
||||||
|
|
||||||
|
belongs_to :build
|
||||||
|
belongs_to :project
|
||||||
|
|
||||||
|
mount_uploader :file, JobArtifactUploader
|
||||||
|
|
||||||
|
enum file_type: {
|
||||||
|
archive: 1,
|
||||||
|
metadata: 2,
|
||||||
|
trace: 3
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
class BuildTraceChunk < ActiveRecord::Base
|
||||||
|
self.table_name = 'ci_build_trace_chunks'
|
||||||
|
|
||||||
|
belongs_to :build
|
||||||
|
end
|
||||||
|
|
||||||
|
class Project < ActiveRecord::Base
|
||||||
|
self.table_name = 'projects'
|
||||||
|
|
||||||
|
has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build'
|
||||||
|
end
|
||||||
|
|
||||||
|
def perform(start_id, stop_id)
|
||||||
|
BackgroundMigration::ArchiveLegacyTraces::Build
|
||||||
|
.finished
|
||||||
|
.without_new_traces
|
||||||
|
.where(id: (start_id..stop_id)).find_each do |build|
|
||||||
|
build.trace.archive!
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,60 @@
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do
|
||||||
|
let(:namespaces) { table(:namespaces) }
|
||||||
|
let(:projects) { table(:projects) }
|
||||||
|
let(:builds) { table(:ci_builds) }
|
||||||
|
let(:job_artifacts) { table(:ci_job_artifacts) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
|
||||||
|
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
|
||||||
|
build = builds.create!(id: 1, project_id: 123, status: 'success')
|
||||||
|
|
||||||
|
@legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path,
|
||||||
|
build.created_at.utc.strftime("%Y_%m"),
|
||||||
|
build.project_id.to_s)
|
||||||
|
|
||||||
|
FileUtils.mkdir_p(@legacy_trace_dir)
|
||||||
|
|
||||||
|
@legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log")
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when trace file exsits at the right place' do
|
||||||
|
before do
|
||||||
|
File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') }
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'correctly archive legacy traces' do
|
||||||
|
expect(job_artifacts.count).to eq(0)
|
||||||
|
expect(File.exist?(@legacy_trace_path)).to be_truthy
|
||||||
|
|
||||||
|
described_class.new.perform(1, 1)
|
||||||
|
|
||||||
|
expect(job_artifacts.count).to eq(1)
|
||||||
|
expect(File.exist?(@legacy_trace_path)).to be_falsy
|
||||||
|
expect(File.read(new_trace_path)).to eq('aiueo')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when trace file does not exsits at the right place' do
|
||||||
|
it 'correctly archive legacy traces' do
|
||||||
|
expect(job_artifacts.count).to eq(0)
|
||||||
|
expect(File.exist?(@legacy_trace_path)).to be_falsy
|
||||||
|
|
||||||
|
described_class.new.perform(1, 1)
|
||||||
|
|
||||||
|
expect(job_artifacts.count).to eq(0)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def new_trace_path
|
||||||
|
job_artifact = job_artifacts.first
|
||||||
|
|
||||||
|
disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s)
|
||||||
|
creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d')
|
||||||
|
|
||||||
|
File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash,
|
||||||
|
creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log')
|
||||||
|
end
|
||||||
|
end
|
45
spec/migrations/archive_legacy_traces_spec.rb
Normal file
45
spec/migrations/archive_legacy_traces_spec.rb
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
require 'spec_helper'
|
||||||
|
require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces')
|
||||||
|
|
||||||
|
describe ArchiveLegacyTraces, :migration do
|
||||||
|
let(:namespaces) { table(:namespaces) }
|
||||||
|
let(:projects) { table(:projects) }
|
||||||
|
let(:builds) { table(:ci_builds) }
|
||||||
|
let(:job_artifacts) { table(:ci_job_artifacts) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
|
||||||
|
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
|
||||||
|
build = builds.create!(id: 1)
|
||||||
|
|
||||||
|
@legacy_trace_path = File.join(
|
||||||
|
Settings.gitlab_ci.builds_path,
|
||||||
|
build.created_at.utc.strftime("%Y_%m"),
|
||||||
|
build.project_id.to_s,
|
||||||
|
"#{job.id}.log"
|
||||||
|
)
|
||||||
|
|
||||||
|
File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') }
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'correctly archive legacy traces' do
|
||||||
|
expect(job_artifacts.count).to eq(0)
|
||||||
|
expect(File.exist?(@legacy_trace_path)).to be_truthy
|
||||||
|
|
||||||
|
migrate!
|
||||||
|
|
||||||
|
expect(job_artifacts.count).to eq(1)
|
||||||
|
expect(File.exist?(@legacy_trace_path)).to be_falsy
|
||||||
|
expect(File.exist?(new_trace_path)).to be_truthy
|
||||||
|
end
|
||||||
|
|
||||||
|
def new_trace_path
|
||||||
|
job_artifact = job_artifacts.first
|
||||||
|
|
||||||
|
disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s)
|
||||||
|
creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d')
|
||||||
|
|
||||||
|
File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
|
||||||
|
creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s)
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in a new issue