Cleanup CI backup => migrate with GitLab
This commit is contained in:
parent
b87ca7500f
commit
ed18e04bb3
|
@ -0,0 +1,30 @@
|
|||
module Backup
|
||||
class Builds
|
||||
attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
|
||||
|
||||
def initialize
|
||||
@app_builds_dir = File.realpath(Rails.root.join('ci/builds'))
|
||||
@backup_dir = GitlabCi.config.backup.path
|
||||
@backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds')
|
||||
end
|
||||
|
||||
# Copy builds from builds directory to backup/builds
|
||||
def dump
|
||||
FileUtils.mkdir_p(backup_builds_dir)
|
||||
FileUtils.cp_r(app_builds_dir, backup_dir)
|
||||
end
|
||||
|
||||
def restore
|
||||
backup_existing_builds_dir
|
||||
|
||||
FileUtils.cp_r(backup_builds_dir, app_builds_dir)
|
||||
end
|
||||
|
||||
def backup_existing_builds_dir
|
||||
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
|
||||
if File.exists?(app_builds_dir)
|
||||
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,32 +0,0 @@
|
|||
module Ci
|
||||
module Backup
|
||||
class Builds
|
||||
attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
|
||||
|
||||
def initialize
|
||||
@app_builds_dir = File.realpath(Rails.root.join('ci/builds'))
|
||||
@backup_dir = GitlabCi.config.backup.path
|
||||
@backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds')
|
||||
end
|
||||
|
||||
# Copy builds from builds directory to backup/builds
|
||||
def dump
|
||||
FileUtils.mkdir_p(backup_builds_dir)
|
||||
FileUtils.cp_r(app_builds_dir, backup_dir)
|
||||
end
|
||||
|
||||
def restore
|
||||
backup_existing_builds_dir
|
||||
|
||||
FileUtils.cp_r(backup_builds_dir, app_builds_dir)
|
||||
end
|
||||
|
||||
def backup_existing_builds_dir
|
||||
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
|
||||
if File.exists?(app_builds_dir)
|
||||
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,94 +0,0 @@
|
|||
require 'yaml'
|
||||
|
||||
module Ci
|
||||
module Backup
|
||||
class Database
|
||||
attr_reader :config, :db_dir
|
||||
|
||||
def initialize
|
||||
@config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
|
||||
@db_dir = File.join(GitlabCi.config.backup.path, 'db')
|
||||
FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
|
||||
end
|
||||
|
||||
def dump
|
||||
success = case config["adapter"]
|
||||
when /^mysql/ then
|
||||
$progress.print "Dumping MySQL database #{config['database']} ... "
|
||||
system('mysqldump', *mysql_args, config['database'], out: db_file_name)
|
||||
when "postgresql" then
|
||||
$progress.print "Dumping PostgreSQL database #{config['database']} ... "
|
||||
pg_env
|
||||
system('pg_dump', config['database'], out: db_file_name)
|
||||
end
|
||||
report_success(success)
|
||||
abort 'Backup failed' unless success
|
||||
end
|
||||
|
||||
def restore
|
||||
success = case config["adapter"]
|
||||
when /^mysql/ then
|
||||
$progress.print "Restoring MySQL database #{config['database']} ... "
|
||||
system('mysql', *mysql_args, config['database'], in: db_file_name)
|
||||
when "postgresql" then
|
||||
$progress.print "Restoring PostgreSQL database #{config['database']} ... "
|
||||
# Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
|
||||
# statements like MySQL.
|
||||
drop_all_tables
|
||||
drop_all_postgres_sequences
|
||||
pg_env
|
||||
system('psql', config['database'], '-f', db_file_name)
|
||||
end
|
||||
report_success(success)
|
||||
abort 'Restore failed' unless success
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def db_file_name
|
||||
File.join(db_dir, 'database.sql')
|
||||
end
|
||||
|
||||
def mysql_args
|
||||
args = {
|
||||
'host' => '--host',
|
||||
'port' => '--port',
|
||||
'socket' => '--socket',
|
||||
'username' => '--user',
|
||||
'encoding' => '--default-character-set',
|
||||
'password' => '--password'
|
||||
}
|
||||
args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
|
||||
end
|
||||
|
||||
def pg_env
|
||||
ENV['PGUSER'] = config["username"] if config["username"]
|
||||
ENV['PGHOST'] = config["host"] if config["host"]
|
||||
ENV['PGPORT'] = config["port"].to_s if config["port"]
|
||||
ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
|
||||
end
|
||||
|
||||
def report_success(success)
|
||||
if success
|
||||
$progress.puts '[DONE]'.green
|
||||
else
|
||||
$progress.puts '[FAILED]'.red
|
||||
end
|
||||
end
|
||||
|
||||
def drop_all_tables
|
||||
connection = ActiveRecord::Base.connection
|
||||
connection.tables.each do |table|
|
||||
connection.drop_table(table)
|
||||
end
|
||||
end
|
||||
|
||||
def drop_all_postgres_sequences
|
||||
connection = ActiveRecord::Base.connection
|
||||
connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence|
|
||||
connection.execute("DROP SEQUENCE #{sequence['relname']}")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,158 +0,0 @@
|
|||
module Ci
|
||||
module Backup
|
||||
class Manager
|
||||
def pack
|
||||
# saving additional informations
|
||||
s = {}
|
||||
s[:db_version] = "#{ActiveRecord::Migrator.current_version}"
|
||||
s[:backup_created_at] = Time.now
|
||||
s[:gitlab_version] = GitlabCi::VERSION
|
||||
s[:tar_version] = tar_version
|
||||
tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz"
|
||||
|
||||
Dir.chdir(GitlabCi.config.backup.path) do
|
||||
File.open("#{GitlabCi.config.backup.path}/backup_information.yml",
|
||||
"w+") do |file|
|
||||
file << s.to_yaml.gsub(/^---\n/,'')
|
||||
end
|
||||
|
||||
FileUtils.chmod(0700, ["db", "builds"])
|
||||
|
||||
# create archive
|
||||
$progress.print "Creating backup archive: #{tar_file} ... "
|
||||
orig_umask = File.umask(0077)
|
||||
if Kernel.system('tar', '-czf', tar_file, *backup_contents)
|
||||
$progress.puts "done".green
|
||||
else
|
||||
puts "creating archive #{tar_file} failed".red
|
||||
abort 'Backup failed'
|
||||
end
|
||||
File.umask(orig_umask)
|
||||
|
||||
upload(tar_file)
|
||||
end
|
||||
end
|
||||
|
||||
def upload(tar_file)
|
||||
remote_directory = GitlabCi.config.backup.upload.remote_directory
|
||||
$progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
|
||||
|
||||
connection_settings = GitlabCi.config.backup.upload.connection
|
||||
if connection_settings.blank?
|
||||
$progress.puts "skipped".yellow
|
||||
return
|
||||
end
|
||||
|
||||
connection = ::Fog::Storage.new(connection_settings)
|
||||
directory = connection.directories.get(remote_directory)
|
||||
|
||||
if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
|
||||
multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size)
|
||||
$progress.puts "done".green
|
||||
else
|
||||
puts "uploading backup to #{remote_directory} failed".red
|
||||
abort 'Backup failed'
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup
|
||||
$progress.print "Deleting tmp directories ... "
|
||||
|
||||
backup_contents.each do |dir|
|
||||
next unless File.exist?(File.join(GitlabCi.config.backup.path, dir))
|
||||
|
||||
if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir))
|
||||
$progress.puts "done".green
|
||||
else
|
||||
puts "deleting tmp directory '#{dir}' failed".red
|
||||
abort 'Backup failed'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def remove_old
|
||||
# delete backups
|
||||
$progress.print "Deleting old backups ... "
|
||||
keep_time = GitlabCi.config.backup.keep_time.to_i
|
||||
|
||||
if keep_time > 0
|
||||
removed = 0
|
||||
|
||||
Dir.chdir(GitlabCi.config.backup.path) do
|
||||
file_list = Dir.glob('*_gitlab_ci_backup.tar.gz')
|
||||
file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ }
|
||||
file_list.sort.each do |timestamp|
|
||||
if Time.at(timestamp) < (Time.now - keep_time)
|
||||
if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz))
|
||||
removed += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
$progress.puts "done. (#{removed} removed)".green
|
||||
else
|
||||
$progress.puts "skipping".yellow
|
||||
end
|
||||
end
|
||||
|
||||
def unpack
|
||||
Dir.chdir(GitlabCi.config.backup.path)
|
||||
|
||||
# check for existing backups in the backup dir
|
||||
file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i }
|
||||
puts "no backups found" if file_list.count == 0
|
||||
|
||||
if file_list.count > 1 && ENV["BACKUP"].nil?
|
||||
puts "Found more than one backup, please specify which one you want to restore:"
|
||||
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
|
||||
exit 1
|
||||
end
|
||||
|
||||
tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz")
|
||||
|
||||
unless File.exists?(tar_file)
|
||||
puts "The specified backup doesn't exist!"
|
||||
exit 1
|
||||
end
|
||||
|
||||
$progress.print "Unpacking backup ... "
|
||||
|
||||
unless Kernel.system(*%W(tar -xzf #{tar_file}))
|
||||
puts "unpacking backup failed".red
|
||||
exit 1
|
||||
else
|
||||
$progress.puts "done".green
|
||||
end
|
||||
|
||||
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
|
||||
|
||||
# restoring mismatching backups can lead to unexpected problems
|
||||
if settings[:gitlab_version] != GitlabCi::VERSION
|
||||
puts "GitLab CI version mismatch:".red
|
||||
puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red
|
||||
puts " Please switch to the following version and try again:".red
|
||||
puts " version: #{settings[:gitlab_version]}".red
|
||||
puts
|
||||
puts "Hint: git checkout v#{settings[:gitlab_version]}"
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
def tar_version
|
||||
tar_version = `tar --version`
|
||||
tar_version.force_encoding('locale').split("\n").first
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def backup_contents
|
||||
["db", "builds", "backup_information.yml"]
|
||||
end
|
||||
|
||||
def settings
|
||||
@settings ||= YAML.load_file("backup_information.yml")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,62 +0,0 @@
|
|||
namespace :ci do
|
||||
namespace :backup do
|
||||
|
||||
desc "GITLAB | Create a backup of the GitLab CI database"
|
||||
task create: :environment do
|
||||
configure_cron_mode
|
||||
|
||||
$progress.puts "Dumping database ... ".blue
|
||||
Ci::Backup::Database.new.dump
|
||||
$progress.puts "done".green
|
||||
|
||||
$progress.puts "Dumping builds ... ".blue
|
||||
Ci::Backup::Builds.new.dump
|
||||
$progress.puts "done".green
|
||||
|
||||
backup = Ci::Backup::Manager.new
|
||||
backup.pack
|
||||
backup.cleanup
|
||||
backup.remove_old
|
||||
end
|
||||
|
||||
desc "GITLAB | Restore a previously created backup"
|
||||
task restore: :environment do
|
||||
configure_cron_mode
|
||||
|
||||
backup = Ci::Backup::Manager.new
|
||||
backup.unpack
|
||||
|
||||
$progress.puts "Restoring database ... ".blue
|
||||
Ci::Backup::Database.new.restore
|
||||
$progress.puts "done".green
|
||||
|
||||
$progress.puts "Restoring builds ... ".blue
|
||||
Ci::Backup::Builds.new.restore
|
||||
$progress.puts "done".green
|
||||
|
||||
backup.cleanup
|
||||
end
|
||||
|
||||
def configure_cron_mode
|
||||
if ENV['CRON']
|
||||
# We need an object we can say 'puts' and 'print' to; let's use a
|
||||
# StringIO.
|
||||
require 'stringio'
|
||||
$progress = StringIO.new
|
||||
else
|
||||
$progress = $stdout
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Disable colors for CRON
|
||||
unless STDOUT.isatty
|
||||
module Colored
|
||||
extend self
|
||||
|
||||
def colorize(string, options={})
|
||||
string
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -11,6 +11,7 @@ namespace :gitlab do
|
|||
Rake::Task["gitlab:backup:db:create"].invoke
|
||||
Rake::Task["gitlab:backup:repo:create"].invoke
|
||||
Rake::Task["gitlab:backup:uploads:create"].invoke
|
||||
Rake::Task["gitlab:backup:builds:create"].invoke
|
||||
|
||||
backup = Backup::Manager.new
|
||||
backup.pack
|
||||
|
@ -30,6 +31,7 @@ namespace :gitlab do
|
|||
Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
|
||||
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
|
||||
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
|
||||
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
|
||||
Rake::Task["gitlab:shell:setup"].invoke
|
||||
|
||||
backup.cleanup
|
||||
|
@ -73,6 +75,25 @@ namespace :gitlab do
|
|||
end
|
||||
end
|
||||
|
||||
namespace :builds do
|
||||
task create: :environment do
|
||||
$progress.puts "Dumping builds ... ".blue
|
||||
|
||||
if ENV["SKIP"] && ENV["SKIP"].include?("builds")
|
||||
$progress.puts "[SKIPPED]".cyan
|
||||
else
|
||||
Backup::Builds.new.dump
|
||||
$progress.puts "done".green
|
||||
end
|
||||
end
|
||||
|
||||
task restore: :environment do
|
||||
$progress.puts "Restoring builds ... ".blue
|
||||
Backup::Builds.new.restore
|
||||
$progress.puts "done".green
|
||||
end
|
||||
end
|
||||
|
||||
namespace :uploads do
|
||||
task create: :environment do
|
||||
$progress.puts "Dumping uploads ... ".blue
|
||||
|
|
|
@ -54,6 +54,7 @@ describe 'gitlab:app namespace rake task' do
|
|||
and_return({ gitlab_version: gitlab_version })
|
||||
expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke)
|
||||
expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
|
||||
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
|
||||
expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
|
||||
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
|
||||
end
|
||||
|
@ -117,6 +118,7 @@ describe 'gitlab:app namespace rake task' do
|
|||
expect(tar_contents).to match('db/')
|
||||
expect(tar_contents).to match('uploads/')
|
||||
expect(tar_contents).to match('repositories/')
|
||||
expect(tar_contents).to match('builds/')
|
||||
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories)\/$/)
|
||||
end
|
||||
|
||||
|
@ -163,6 +165,7 @@ describe 'gitlab:app namespace rake task' do
|
|||
|
||||
expect(tar_contents).to match('db/')
|
||||
expect(tar_contents).to match('uploads/')
|
||||
expect(tar_contents).to match('builds/')
|
||||
expect(tar_contents).not_to match('repositories/')
|
||||
end
|
||||
|
||||
|
@ -173,6 +176,7 @@ describe 'gitlab:app namespace rake task' do
|
|||
|
||||
expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
|
||||
expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
|
||||
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
|
||||
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
|
||||
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue