Create CI migration task on GitLab side
This commit is contained in:
Kamil Trzcinski
parent
45a105b127
commit
265ad515c6
|
|
@ -0,0 +1,29 @@
|
|||
module Ci
|
||||
module Migrate
|
||||
class Builds
|
||||
attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
|
||||
|
||||
def initialize
|
||||
@app_builds_dir = Settings.gitlab_ci.builds_path
|
||||
@backup_dir = Gitlab.config.backup.path
|
||||
@backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
|
||||
end
|
||||
|
||||
def restore
|
||||
backup_existing_builds_dir
|
||||
|
||||
FileUtils.mkdir_p(app_builds_dir, mode: 0700)
|
||||
unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
|
||||
abort 'Restore failed'.red
|
||||
end
|
||||
end
|
||||
|
||||
def backup_existing_builds_dir
|
||||
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
|
||||
if File.exists?(app_builds_dir)
|
||||
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -9,32 +9,32 @@ module Ci
|
|||
@config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
|
||||
end
|
||||
|
||||
def restore(ci_dump)
|
||||
puts 'Deleting all CI related data ... '
|
||||
truncate_ci_tables
|
||||
def restore
|
||||
decompress_rd, decompress_wr = IO.pipe
|
||||
decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
|
||||
decompress_wr.close
|
||||
|
||||
puts 'Restoring CI data ... '
|
||||
case config["adapter"]
|
||||
when /^mysql/ then
|
||||
print "Restoring MySQL database #{config['database']} ... "
|
||||
# Workaround warnings from MySQL 5.6 about passwords on cmd line
|
||||
ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
|
||||
system('mysql', *mysql_args, config['database'], in: ci_dump)
|
||||
when "postgresql" then
|
||||
puts "Restoring PostgreSQL database #{config['database']} ... "
|
||||
pg_env
|
||||
system('psql', config['database'], '-f', ci_dump)
|
||||
end
|
||||
restore_pid = case config["adapter"]
|
||||
when /^mysql/ then
|
||||
$progress.print "Restoring MySQL database #{config['database']} ... "
|
||||
# Workaround warnings from MySQL 5.6 about passwords on cmd line
|
||||
ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
|
||||
spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
|
||||
when "postgresql" then
|
||||
$progress.print "Restoring PostgreSQL database #{config['database']} ... "
|
||||
pg_env
|
||||
spawn('psql', config['database'], in: decompress_rd)
|
||||
end
|
||||
decompress_rd.close
|
||||
|
||||
success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
|
||||
abort 'Restore failed' unless success
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def truncate_ci_tables
|
||||
c = ActiveRecord::Base.connection
|
||||
c.tables.select { |t| t.start_with?('ci_') }.each do |table|
|
||||
puts "Deleting data from #{table}..."
|
||||
c.execute("DELETE FROM #{table}")
|
||||
end
|
||||
def db_file_name
|
||||
File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
|
||||
end
|
||||
|
||||
def mysql_args
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
module Ci
|
||||
module Migrate
|
||||
class Manager
|
||||
VERSION = '8.0.0.pre'
|
||||
|
||||
def cleanup
|
||||
$progress.print "Deleting tmp directories ... "
|
||||
|
||||
backup_contents.each do |dir|
|
||||
next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
|
||||
|
||||
if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
|
||||
$progress.puts "done".green
|
||||
else
|
||||
puts "deleting tmp directory '#{dir}' failed".red
|
||||
abort 'Backup failed'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def unpack
|
||||
Dir.chdir(Gitlab.config.backup.path)
|
||||
|
||||
# check for existing backups in the backup dir
|
||||
file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
|
||||
puts "no backups found" if file_list.count == 0
|
||||
|
||||
if file_list.count > 1 && ENV["BACKUP"].nil?
|
||||
puts "Found more than one backup, please specify which one you want to restore:"
|
||||
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
|
||||
exit 1
|
||||
end
|
||||
|
||||
tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
|
||||
|
||||
unless File.exists?(tar_file)
|
||||
puts "The specified CI backup doesn't exist!"
|
||||
exit 1
|
||||
end
|
||||
|
||||
$progress.print "Unpacking backup ... "
|
||||
|
||||
unless Kernel.system(*%W(tar -xf #{tar_file}))
|
||||
puts "unpacking backup failed".red
|
||||
exit 1
|
||||
else
|
||||
$progress.puts "done".green
|
||||
end
|
||||
|
||||
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
|
||||
|
||||
# restoring mismatching backups can lead to unexpected problems
|
||||
if settings[:gitlab_version] != VERSION
|
||||
puts "GitLab CI version mismatch:".red
|
||||
puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def backup_contents
|
||||
["db", "builds", "backup_information.yml"]
|
||||
end
|
||||
|
||||
def settings
|
||||
@settings ||= YAML.load_file("backup_information.yml")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -1,40 +1,49 @@
|
|||
namespace :ci do
|
||||
desc 'GitLab | Import and migrate CI database'
|
||||
task migrate: :environment do
|
||||
warn_user_is_not_gitlab
|
||||
configure_cron_mode
|
||||
|
||||
unless ENV['force'] == 'yes'
|
||||
puts "This will truncate all CI tables and restore it from provided backup."
|
||||
puts "You will lose any previous CI data stored in the database."
|
||||
puts 'This will remove all CI related data and restore it from the provided backup.'
|
||||
ask_to_continue
|
||||
puts ""
|
||||
puts ''
|
||||
end
|
||||
|
||||
Rake::Task["ci:migrate:db"].invoke
|
||||
Rake::Task["ci:migrate:autoincrements"].invoke
|
||||
Rake::Task["ci:migrate:tags"].invoke
|
||||
Rake::Task["ci:migrate:services"].invoke
|
||||
migrate = Ci::Migrate::Manager.new
|
||||
migrate.unpack
|
||||
|
||||
Rake::Task['ci:migrate:db'].invoke
|
||||
Rake::Task['ci:migrate:builds'].invoke
|
||||
Rake::Task['ci:migrate:tags'].invoke
|
||||
Rake::Task['ci:migrate:services'].invoke
|
||||
|
||||
migrate.cleanup
|
||||
end
|
||||
|
||||
namespace :migrate do
|
||||
desc 'GitLab | Import CI database'
|
||||
task db: :environment do
|
||||
if ENV["CI_DUMP"].nil?
|
||||
puts "No CI SQL dump specified:"
|
||||
puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
|
||||
exit 1
|
||||
end
|
||||
configure_cron_mode
|
||||
$progress.puts 'Restoring database ... '.blue
|
||||
Ci::Migrate::Database.new.restore
|
||||
$progress.puts 'done'.green
|
||||
end
|
||||
|
||||
ci_dump = ENV["CI_DUMP"]
|
||||
unless File.exists?(ci_dump)
|
||||
puts "The specified sql dump doesn't exist!"
|
||||
exit 1
|
||||
end
|
||||
|
||||
::Ci::Migrate::Database.new.restore(ci_dump)
|
||||
desc 'GitLab | Import CI builds'
|
||||
task builds: :environment do
|
||||
configure_cron_mode
|
||||
$progress.puts 'Restoring builds ... '.blue
|
||||
Ci::Migrate::Builds.new.restore
|
||||
$progress.puts 'done'.green
|
||||
end
|
||||
|
||||
desc 'GitLab | Migrate CI tags'
|
||||
task tags: :environment do
|
||||
configure_cron_mode
|
||||
$progress.puts 'Migrating tags ... '.blue
|
||||
::Ci::Migrate::Tags.new.restore
|
||||
$progress.puts 'done'.green
|
||||
end
|
||||
|
||||
desc 'GitLab | Migrate CI auto-increments'
|
||||
|
|
@ -56,8 +65,10 @@ namespace :ci do
|
|||
|
||||
desc 'GitLab | Migrate CI services'
|
||||
task services: :environment do
|
||||
$progress.puts 'Migrating services ... '.blue
|
||||
c = ActiveRecord::Base.connection
|
||||
c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
|
||||
$progress.puts 'done'.green
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue