Merge branch 'incremental-backups' into 'master'

Incremental backups

See merge request gitlab-org/gitlab-ce!24035

doc/raketasks/backup_restore.md

@@ -195,6 +195,26 @@ To use the `copy` strategy instead of the default streaming strategy, specify
 sudo gitlab-rake gitlab:backup:create STRATEGY=copy
 ```
 
+### Backup filename
+
+By default a backup file is created according to the specification in [the Backup timestamp](#backup-timestamp) section above. You can however override the `[TIMESTAMP]` part of the filename by setting the `BACKUP` environment variable. For example:
+
+```sh
+sudo gitlab-rake gitlab:backup:create BACKUP=dump
+```
+
+The resulting file will then be `dump_gitlab_backup.tar`. This is useful for systems that make use of rsync and incremental backups, and will result in considerably faster transfer speeds.
+
+### Rsyncable
+
+To make sure the generated archive is intelligently transferable by rsync, the `GZIP_RSYNCABLE=yes` option can be set. This will set the `--rsyncable` option to `gzip`. This is only useful in combination with setting [the Backup filename option](#backup-filename).
+
+Note that the `--rsyncable` option in `gzip` is not guaranteed to be available on all distributions. To verify that it is available in your distribution you can run `gzip --help` or consult the man pages.
+
+```sh
+sudo gitlab-rake gitlab:backup:create BACKUP=dump GZIP_RSYNCABLE=yes
+```
+
 ### Excluding specific directories from the backup
 
 You can choose what should be exempt from the backup up by adding the environment variable `SKIP`.

lib/backup/database.rb

@@ -4,6 +4,7 @@ require 'yaml'
 
 module Backup
   class Database
+    include Backup::Helper
     attr_reader :progress
     attr_reader :config, :db_file_name
 
@@ -17,7 +18,7 @@ module Backup
       FileUtils.mkdir_p(File.dirname(db_file_name))
       FileUtils.rm_f(db_file_name)
       compress_rd, compress_wr = IO.pipe
-      compress_pid = spawn(*%w(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
+      compress_pid = spawn(gzip_cmd, in: compress_rd, out: [db_file_name, 'w', 0600])
       compress_rd.close
 
       dump_pid =

lib/backup/files.rb

@@ -31,10 +31,10 @@ module Backup
           raise Backup::Error, 'Backup failed'
         end
 
-        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), gzip_cmd], out: [backup_tarball, 'w', 0600])
         FileUtils.rm_rf(@backup_files_dir)
       else
-        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), gzip_cmd], out: [backup_tarball, 'w', 0600])
       end
     end
 

lib/backup/helper.rb

@@ -29,5 +29,13 @@ module Backup
       EOS
       raise message
     end
+
+    def gzip_cmd
+      @gzip_cmd ||= if ENV['GZIP_RSYNCABLE'] == 'yes'
+                      "gzip --rsyncable -c -1"
+                    else
+                      "gzip -c -1"
+                    end
+    end
   end
 end

lib/backup/manager.rb

@@ -235,7 +235,11 @@ module Backup
     end
 
     def tar_file
-      @tar_file ||= "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
+      @tar_file ||= if ENV['BACKUP']
+                      ENV['BACKUP'] + "#{FILE_NAME_SUFFIX}"
+                    else
+                      "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
+                    end
     end
 
     def backup_information