gitlab-org--gitlab-foss/lib/backup/repository.rb

require 'yaml'
require_relative 'helper'

module Backup
  class Repository
    include Backup::Helper
    # rubocop:disable Metrics/AbcSize

    attr_reader :progress

    def initialize(progress)
      @progress = progress
    end

    def dump
      prepare

      Project.find_each(batch_size: 1000) do |project|
        progress.print " * #{display_repo_path(project)} ... "
        path_to_project_repo = path_to_repo(project)
        path_to_project_bundle = path_to_bundle(project)

        # Create namespace dir or hashed path if missing
        if project.hashed_storage?(:repository)
          FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
        else
          FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
        end

        if empty_repo?(project)
          progress.puts "[SKIPPED]".color(:cyan)
        else
          in_path(path_to_project_repo) do |dir|
            FileUtils.mkdir_p(path_to_tars(project))
            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
            output, status = Gitlab::Popen.popen(cmd)

            unless status.zero?
              progress_warn(project, cmd.join(' '), output)
            end
          end

          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
          output, status = Gitlab::Popen.popen(cmd)

          if status.zero?
            progress.puts "[DONE]".color(:green)
          else
            progress_warn(project, cmd.join(' '), output)
          end
        end

        wiki = ProjectWiki.new(project)
        path_to_wiki_repo = path_to_repo(wiki)
        path_to_wiki_bundle = path_to_bundle(wiki)

        if File.exist?(path_to_wiki_repo)
          progress.print " * #{display_repo_path(wiki)} ... "

          if empty_repo?(wiki)
            progress.puts " [SKIPPED]".color(:cyan)
          else
            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
            output, status = Gitlab::Popen.popen(cmd)
            if status.zero?
              progress.puts " [DONE]".color(:green)
            else
              progress_warn(wiki, cmd.join(' '), output)
            end
          end
        end
      end
    end

    def prepare_directories
      Gitlab.config.repositories.storages.each do |name, repository_storage|
        delete_all_repositories(name, repository_storage)
      end
    end

    def delete_all_repositories(name, repository_storage)
      gitaly_migrate(:delete_all_repositories) do |is_enabled|
        if is_enabled
          Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
        else
          local_delete_all_repositories(name, repository_storage)
        end
      end
    end

    def local_delete_all_repositories(name, repository_storage)
      path = repository_storage.legacy_disk_path
      return unless File.exist?(path)

      # Move all files in the existing repos directory except . and .. to
      # repositories.old.<timestamp> directory
      bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
      FileUtils.mkdir_p(bk_repos_path, mode: 0700)
      files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]

      begin
        FileUtils.mv(files, bk_repos_path)
      rescue Errno::EACCES
        access_denied_error(path)
      rescue Errno::EBUSY
        resource_busy_error(path)
      end
    end

    def restore_custom_hooks(project)
      # TODO: Need to find a way to do this for gitaly
      # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
      in_path(path_to_tars(project)) do |dir|
        path_to_project_repo = path_to_repo(project)
        cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})

        output, status = Gitlab::Popen.popen(cmd)
        unless status.zero?
          progress_warn(project, cmd.join(' '), output)
        end
      end
    end

    def restore
      prepare_directories
      gitlab_shell = Gitlab::Shell.new

      Project.find_each(batch_size: 1000) do |project|
        progress.print " * #{project.full_path} ... "
        path_to_project_bundle = path_to_bundle(project)
        project.ensure_storage_path_exists

        restore_repo_success = nil
        if File.exist?(path_to_project_bundle)
          begin
            project.repository.create_from_bundle path_to_project_bundle
            restore_repo_success = true
          rescue => e
            restore_repo_success = false
            progress.puts "Error: #{e}".color(:red)
          end
        else
          restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
        end

        if restore_repo_success
          progress.puts "[DONE]".color(:green)
        else
          progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
        end

        restore_custom_hooks(project)

        wiki = ProjectWiki.new(project)
        path_to_wiki_bundle = path_to_bundle(wiki)

        if File.exist?(path_to_wiki_bundle)
          progress.print " * #{wiki.full_path} ... "
          begin
            wiki.repository.create_from_bundle(path_to_wiki_bundle)
            progress.puts "[DONE]".color(:green)
          rescue => e
            progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
            progress.puts "Error #{e}".color(:red)
          end
        end
      end
    end
    # rubocop:enable Metrics/AbcSize

    protected

    def path_to_repo(project)
      project.repository.path_to_repo
    end

    def path_to_bundle(project)
      File.join(backup_repos_path, project.disk_path + '.bundle')
    end

    def path_to_tars(project, dir = nil)
      path = File.join(backup_repos_path, project.disk_path)

      if dir
        File.join(path, "#{dir}.tar")
      else
        path
      end
    end

    def backup_repos_path
      File.join(Gitlab.config.backup.path, 'repositories')
    end

    def in_path(path)
      return unless Dir.exist?(path)

      dir_entries = Dir.entries(path)

      if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
        yield('custom_hooks')
      end
    end

    def prepare
      FileUtils.rm_rf(backup_repos_path)
      # Ensure the parent dir of backup_repos_path exists
      FileUtils.mkdir_p(Gitlab.config.backup.path)
      # Fail if somebody raced to create backup_repos_path before us
      FileUtils.mkdir(backup_repos_path, mode: 0700)
    end

    def silent
      { err: '/dev/null', out: '/dev/null' }
    end

    private

    def progress_warn(project, cmd, output)
      progress.puts "[WARNING] Executing #{cmd}".color(:orange)
      progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
    end

    def empty_repo?(project_or_wiki)
      # Protect against stale caches
      project_or_wiki.repository.expire_emptiness_caches
      project_or_wiki.repository.empty?
    end

    def repository_storage_paths_args
      Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
    end

    def display_repo_path(project)
      project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
    end

    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
      Gitlab::GitalyClient.migrate(method, status: status, &block)
    rescue GRPC::NotFound, GRPC::BadStatus => e
      raise Error, e
    end
  end
end