Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-03-15 03:08:45 +00:00
parent 34e729c8b9
commit c16d7a4f01
47 changed files with 1112 additions and 747 deletions

View File

@ -0,0 +1 @@
<svg enable-background="new 0 0 50 50" viewBox="0 0 50 50" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><linearGradient id="a" gradientUnits="userSpaceOnUse" x1="10.2941" x2="50" y1="17.8247" y2="17.8247"><stop offset="0" stop-color="#f39c63"/><stop offset="1" stop-color="#ef5d4f"/></linearGradient><linearGradient id="b"><stop offset="0" stop-color="#231c4f"/><stop offset="1" stop-color="#0a0430"/></linearGradient><linearGradient id="c" gradientUnits="userSpaceOnUse" x1="16.9118" x2="25" xlink:href="#b" y1="27.2046" y2="27.2046"/><linearGradient id="d" gradientUnits="userSpaceOnUse" x1="27.9412" x2="33.0882" xlink:href="#b" y1="22.0575" y2="22.0575"/><g clip-rule="evenodd" fill-rule="evenodd"><path d="m31.94 41.71-31.94 8.41v-21.23c0-4.73 3.19-8.87 7.77-10.07l19.29-5.08c6.4-1.68 12.65 3.14 12.65 9.75v8.14c0 4.74-3.19 8.88-7.77 10.08z" fill="#2b6af9"/><path d="m50 0v21.23c0 4.73-3.19 8.87-7.77 10.07l-14.79 3.89c-8.67 2.28-17.15-4.26-17.15-13.22v-3.49c0-4.73 3.19-8.87 7.77-10.07z" fill="url(#a)"/><path d="m23.36 36.33 16.35-4.3v-8.16c0-6.83-6.46-11.81-13.06-10.07l-16.35 4.3v8.16c-.01 6.82 6.45 11.8 13.06 10.07z" fill="#fff"/><circle cx="20.96" cy="27.2" fill="url(#c)" r="4.04"/><circle cx="30.51" cy="22.06" fill="url(#d)" r="2.57"/></g></svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -0,0 +1 @@
<svg enable-background="new 0 0 0 0" viewBox="0 0 800 780" xmlns="http://www.w3.org/2000/svg"><path d="m594.4 737.87c-.75-1.93-1.86-3.7-3.34-5.29-1.48-1.6-3.29-2.86-5.44-3.8-2.15-.93-4.62-1.4-7.41-1.4s-5.26.47-7.41 1.4c-2.14.94-3.96 2.21-5.43 3.8-1.48 1.59-2.59 3.36-3.34 5.29s-1.13 3.91-1.13 5.95v.96c0 1.96.36 3.91 1.1 5.86.73 1.96 1.82 3.74 3.28 5.35s3.27 2.91 5.43 3.89c2.17.98 4.67 1.46 7.5 1.46s5.33-.49 7.5-1.46c2.16-.98 3.98-2.28 5.43-3.89 1.46-1.61 2.55-3.4 3.28-5.35s1.1-3.91 1.1-5.86v-.96c.01-2.04-.37-4.02-1.12-5.95zm-4.01 13.45c-1.15 2.1-2.78 3.77-4.86 5.02-2.09 1.25-4.52 1.89-7.32 1.89-2.79 0-5.24-.63-7.32-1.89-2.09-1.25-3.71-2.93-4.86-5.02s-1.73-4.42-1.73-6.97c0-2.63.58-4.99 1.73-7.09 1.15-2.09 2.77-3.74 4.86-4.96 2.08-1.21 4.52-1.82 7.32-1.82 2.79 0 5.23.61 7.32 1.82 2.08 1.22 3.71 2.87 4.86 4.96 1.15 2.1 1.73 4.46 1.73 7.09 0 2.56-.58 4.88-1.73 6.97z" fill="#f79200"/><path d="m584.11 746.03c1.42-1.12 2.12-2.73 2.12-4.84s-.71-3.73-2.12-4.85c-1.42-1.11-3.4-1.67-5.95-1.67h-2.55-2.37-2.12v18.66h4.49v-5.62h2.55c.09 0 .17-.01.25-.01l3.81 5.64h5.1l-4.69-6.45c.54-.25 1.04-.52 1.48-.86zm-8.5-7.6h2.85c1.1 0 1.9.23 2.43.69s.79 1.15.79 2.07-.26 1.6-.79 2.06-1.33.68-2.43.68h-2.85z" fill="#f79200"/><path d="m333.8 753.41-17.62 7.9v-88.45s76.18-50.47 93.76-64.21c46.25-36.15 101.39-92.54 101.39-155.86v-22.33l40.76-32.62h-235.91v-83.73h280.18v138.69c0 180.72-251.84 295.81-262.56 300.61zm-17.53-638.54c21.22 2.88 121.7 20.25 195.07 97.32v62.85h85.03v-94.02l-10-11.68c-104.42-122.2-259.7-137.72-266.27-138.32l-3.92-.36v84.23c.03-.01.06-.01.09-.02z" fill="#ffbe12"/><path d="m316.18 397.85h-280.18v-216.82l9.99-11.68c104.42-122.21 259.7-137.73 266.25-138.33l3.94-.36v84.23c-21.06 2.97-122.24 20.87-195.17 97.32v74.53l-36.5 27.39h231.66v83.72zm-93.77 210.81c-46.27-36.15-101.4-87.26-101.4-143.31l20.24-28.41h-105.25v28.41c0 171.17 251.84 283.27 262.56 288.07l17.63 7.9v-88.45c-.01 0-76.2-50.47-93.78-64.21z" fill="#f79200"/></svg>

After

Width:  |  Height:  |  Size: 1.9 KiB

View File

@ -284,9 +284,9 @@ export const AUTO_DEVOPS_ENABLED_ALERT_DISMISSED_STORAGE_KEY =
// https://gitlab.com/gitlab-org/gitlab/-/issues/346899
export const TEMP_PROVIDER_LOGOS = {
Kontra: {
svg: '/assets/illustrations/vulnerability/kontra-logo.svg',
svg: '/assets/vulnerability/kontra-logo.svg',
},
[__('Secure Code Warrior')]: {
svg: '/assets/illustrations/vulnerability/scw-logo.svg',
svg: '/assets/vulnerability/scw-logo.svg',
},
};

View File

@ -1,13 +1,10 @@
query getSecurityTrainingVulnerability($id: ID!) {
vulnerability(id: $id) @client {
query getSecurityTrainingUrls($projectFullPath: ID!, $identifierExternalIds: [String!]!) {
project(fullPath: $projectFullPath) {
id
identifiers {
externalType
}
securityTrainingUrls {
securityTrainingUrls(identifierExternalIds: $identifierExternalIds) {
name
url
status
url
}
}
}

View File

@ -42,6 +42,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
push_frontend_feature_flag(:refactor_mr_widgets_extensions, project, default_enabled: :yaml)
push_frontend_feature_flag(:rebase_without_ci_ui, project, default_enabled: :yaml)
push_frontend_feature_flag(:markdown_continue_lists, project, default_enabled: :yaml)
push_frontend_feature_flag(:secure_vulnerability_training, project, default_enabled: :yaml)
# Usage data feature flags
push_frontend_feature_flag(:users_expanding_widgets_usage_data, project, default_enabled: :yaml)
push_frontend_feature_flag(:diff_settings_usage_data, default_enabled: :yaml)

View File

@ -66,20 +66,20 @@ module Users
# rubocop: disable CodeReuse/ActiveRecord
def migrate_issues
user.issues.update_all(author_id: ghost_user.id)
Issue.where(last_edited_by_id: user.id).update_all(last_edited_by_id: ghost_user.id)
batched_migrate(Issue, :author_id)
batched_migrate(Issue, :last_edited_by_id)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def migrate_merge_requests
user.merge_requests.update_all(author_id: ghost_user.id)
MergeRequest.where(merge_user_id: user.id).update_all(merge_user_id: ghost_user.id)
batched_migrate(MergeRequest, :author_id)
batched_migrate(MergeRequest, :merge_user_id)
end
# rubocop: enable CodeReuse/ActiveRecord
def migrate_notes
user.notes.update_all(author_id: ghost_user.id)
batched_migrate(Note, :author_id)
end
def migrate_abuse_reports
@ -96,8 +96,17 @@ module Users
end
def migrate_reviews
user.reviews.update_all(author_id: ghost_user.id)
batched_migrate(Review, :author_id)
end
# rubocop:disable CodeReuse/ActiveRecord
def batched_migrate(base_scope, column)
loop do
update_count = base_scope.where(column => user.id).limit(100).update_all(column => ghost_user.id)
break if update_count == 0
end
end
# rubocop:enable CodeReuse/ActiveRecord
end
end

View File

@ -749,6 +749,38 @@ deploystacks:
- ${PROVIDER}-${STACK}
```
#### Fetch artifacts from a `parallel:matrix` job
You can fetch artifacts from a job created with [`parallel:matrix`](../yaml/index.md#parallelmatrix)
by using the [`dependencies`](../yaml/index.md#dependencies) keyword. Use the job name
as the value for `dependencies` as a string in the form:
```plaintext
<job_name> [<matrix argument 1>, <matrix argument 2>, ... <matrix argument N>]
```
For example, to fetch the artifacts from the job with a `RUBY_VERSION` of `2.7` and
a `PROVIDER` of `aws`:
```yaml
ruby:
image: ruby:${RUBY_VERSION}
parallel:
matrix:
- RUBY_VERSION: ["2.5", "2.6", "2.7", "3.0", "3.1"]
PROVIDER: [aws, gcp]
script: bundle install
deploy:
image: ruby:2.7
stage: deploy
dependencies:
- "ruby: [2.7, aws]"
script: echo hello
```
Quotes around the `dependencies` entry are required.
## Use predefined CI/CD variables to run jobs only in specific pipeline types
You can use [predefined CI/CD variables](../variables/predefined_variables.md) to choose

View File

@ -51,7 +51,7 @@ results. On failure, the analyzer outputs an
## Prerequisites
- [GitLab Runner](../../../ci/runners/index.md) available, with the
[`docker` executor](https://docs.gitlab.com/runner/executors/docker.html).
[`docker` executor](https://docs.gitlab.com/runner/executors/docker.html) on Linux/amd64.
- Target application deployed. For more details, read [Deployment options](#deployment-options).
- DAST runs in the `dast` stage, which must be added manually to your `.gitlab-ci.yml`.

View File

@ -57,7 +57,7 @@ To run SAST jobs, by default, you need GitLab Runner with the
If you're using the shared runners on GitLab.com, this is enabled by default.
WARNING:
Our SAST jobs require a Linux container type. Windows containers are not yet supported.
Our SAST jobs require a Linux/amd64 container type. Windows containers are not yet supported.
WARNING:
If you use your own runners, make sure the Docker version installed

View File

@ -2,14 +2,11 @@
module Backup
class Artifacts < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('artifacts', JobArtifactUploader.root, excludes: ['tmp'])
super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp'])
end
override :human_name
def human_name
_('artifacts')
end

View File

@ -2,14 +2,11 @@
module Backup
class Builds < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('builds', Settings.gitlab_ci.builds_path)
super(progress, 'builds', Settings.gitlab_ci.builds_path)
end
override :human_name
def human_name
_('builds')
end

View File

@ -3,10 +3,10 @@
require 'yaml'
module Backup
class Database
class Database < Task
extend ::Gitlab::Utils::Override
include Backup::Helper
attr_reader :progress
attr_reader :config, :db_file_name
attr_reader :force, :config
IGNORED_ERRORS = [
# Ignore warnings
@ -18,13 +18,14 @@ module Backup
].freeze
IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze
def initialize(progress, filename: nil)
@progress = progress
def initialize(progress, force:)
super(progress)
@config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
@db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
@force = force
end
def dump
override :dump
def dump(db_file_name)
FileUtils.mkdir_p(File.dirname(db_file_name))
FileUtils.rm_f(db_file_name)
compress_rd, compress_wr = IO.pipe
@ -64,12 +65,24 @@ module Backup
raise DatabaseBackupError.new(config, db_file_name) unless success
end
def restore
override :restore
def restore(db_file_name)
unless force
progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
sleep(5)
end
# Drop all tables Load the schema to ensure we don't have any newer tables
# hanging out from a failed upgrade
puts_time 'Cleaning the database ... '.color(:blue)
Rake::Task['gitlab:db:drop_tables'].invoke
puts_time 'done'.color(:green)
decompress_rd, decompress_wr = IO.pipe
decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
decompress_wr.close
status, errors =
status, @errors =
case config[:adapter]
when "postgresql" then
progress.print "Restoring PostgreSQL database #{database} ... "
@ -81,33 +94,47 @@ module Backup
Process.waitpid(decompress_pid)
success = $?.success? && status.success?
if errors.present?
if @errors.present?
progress.print "------ BEGIN ERRORS -----\n".color(:yellow)
progress.print errors.join.color(:yellow)
progress.print @errors.join.color(:yellow)
progress.print "------ END ERRORS -------\n".color(:yellow)
end
report_success(success)
raise Backup::Error, 'Restore failed' unless success
if errors.present?
warning = <<~MSG
There were errors in restoring the schema. This may cause
issues if this results in missing indexes, constraints, or
columns. Please record the errors above and contact GitLab
Support if you have questions:
https://about.gitlab.com/support/
MSG
warn warning.color(:red)
Gitlab::TaskHelpers.ask_to_continue
end
end
def enabled
true
override :pre_restore_warning
def pre_restore_warning
return if force
<<-MSG.strip_heredoc
Be sure to stop Puma, Sidekiq, and any other process that
connects to the database before proceeding. For Omnibus
installs, see the following link for more information:
https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
Before restoring the database, we will remove all existing
tables to avoid future upgrade problems. Be aware that if you have
custom tables in the GitLab database these tables and all data will be
removed.
MSG
end
override :post_restore_warning
def post_restore_warning
return unless @errors.present?
<<-MSG.strip_heredoc
There were errors in restoring the schema. This may cause
issues if this results in missing indexes, constraints, or
columns. Please record the errors above and contact GitLab
Support if you have questions:
https://about.gitlab.com/support/
MSG
end
override :human_name
def human_name
_('database')
end

View File

@ -1,25 +1,27 @@
# frozen_string_literal: true
require 'open3'
require_relative 'helper'
module Backup
class Files
class Files < Task
extend ::Gitlab::Utils::Override
include Backup::Helper
DEFAULT_EXCLUDE = 'lost+found'
attr_reader :name, :backup_tarball, :excludes
attr_reader :name, :excludes
def initialize(progress, name, app_files_dir, excludes: [])
super(progress)
def initialize(name, app_files_dir, excludes: [])
@name = name
@app_files_dir = app_files_dir
@backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
@excludes = [DEFAULT_EXCLUDE].concat(excludes)
end
# Copy files from public/files to backup/files
def dump
override :dump
def dump(backup_tarball)
FileUtils.mkdir_p(Gitlab.config.backup.path)
FileUtils.rm_f(backup_tarball)
@ -35,7 +37,7 @@ module Backup
unless status == 0
puts output
raise_custom_error
raise_custom_error(backup_tarball)
end
tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten
@ -47,11 +49,12 @@ module Backup
end
unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output)
raise_custom_error
raise_custom_error(backup_tarball)
end
end
def restore
override :restore
def restore(backup_tarball)
backup_existing_files_dir
cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]]
@ -61,10 +64,6 @@ module Backup
end
end
def enabled
true
end
def tar
if system(*%w[gtar --version], out: '/dev/null')
# It looks like we can get GNU tar by running 'gtar'
@ -146,7 +145,7 @@ module Backup
end
end
def raise_custom_error
def raise_custom_error(backup_tarball)
raise FileBackupError.new(app_files_realpath, backup_tarball)
end

View File

@ -15,7 +15,7 @@ module Backup
@storage_parallelism = storage_parallelism
end
def start(type)
def start(type, backup_repos_path)
raise Error, 'already started' if started?
command = case type
@ -93,10 +93,6 @@ module Backup
@thread.present?
end
def backup_repos_path
File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
end
def bin_path
File.absolute_path(Gitlab.config.backup.gitaly_backup_path)
end

View File

@ -7,10 +7,11 @@ module Backup
@progress = progress
end
def start(type)
def start(type, backup_repos_path)
raise Error, 'already started' if @type
@type = type
@backup_repos_path = backup_repos_path
case type
when :create
FileUtils.rm_rf(backup_repos_path)
@ -31,7 +32,7 @@ module Backup
backup_restore = BackupRestore.new(
progress,
repository_type.repository_for(container),
backup_repos_path
@backup_repos_path
)
case @type
@ -52,10 +53,6 @@ module Backup
attr_reader :progress
def backup_repos_path
@backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
end
class BackupRestore
attr_accessor :progress, :repository, :backup_repos_path

View File

@ -2,14 +2,11 @@
module Backup
class Lfs < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('lfs', Settings.lfs.storage_path)
super(progress, 'lfs', Settings.lfs.storage_path)
end
override :human_name
def human_name
_('lfs objects')
end

View File

@ -2,37 +2,77 @@
module Backup
class Manager
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry packages].freeze
FOLDERS_TO_BACKUP = %w[repositories db].freeze
FILE_NAME_SUFFIX = '_gitlab_backup.tar'
MANIFEST_NAME = 'backup_information.yml'
TaskDefinition = Struct.new(
:destination_path, # Where the task should put its backup file/dir.
:destination_optional, # `true` if the destination might not exist on a successful backup.
:cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified.
:task,
keyword_init: true
)
attr_reader :progress
def initialize(progress)
def initialize(progress, definitions: nil)
@progress = progress
max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
force = ENV['force'] == 'yes'
@tasks = {
'db' => Database.new(progress),
'repositories' => Repositories.new(progress,
strategy: repository_backup_strategy,
max_concurrency: max_concurrency,
max_storage_concurrency: max_storage_concurrency),
'uploads' => Uploads.new(progress),
'builds' => Builds.new(progress),
'artifacts' => Artifacts.new(progress),
'pages' => Pages.new(progress),
'lfs' => Lfs.new(progress),
'terraform_state' => TerraformState.new(progress),
'registry' => Registry.new(progress),
'packages' => Packages.new(progress)
@definitions = definitions || {
'db' => TaskDefinition.new(
destination_path: 'db/database.sql.gz',
cleanup_path: 'db',
task: Database.new(progress, force: force)
),
'repositories' => TaskDefinition.new(
destination_path: 'repositories',
destination_optional: true,
task: Repositories.new(progress,
strategy: repository_backup_strategy,
max_concurrency: max_concurrency,
max_storage_concurrency: max_storage_concurrency)
),
'uploads' => TaskDefinition.new(
destination_path: 'uploads.tar.gz',
task: Uploads.new(progress)
),
'builds' => TaskDefinition.new(
destination_path: 'builds.tar.gz',
task: Builds.new(progress)
),
'artifacts' => TaskDefinition.new(
destination_path: 'artifacts.tar.gz',
task: Artifacts.new(progress)
),
'pages' => TaskDefinition.new(
destination_path: 'pages.tar.gz',
task: Pages.new(progress)
),
'lfs' => TaskDefinition.new(
destination_path: 'lfs.tar.gz',
task: Lfs.new(progress)
),
'terraform_state' => TaskDefinition.new(
destination_path: 'terraform_state.tar.gz',
task: TerraformState.new(progress)
),
'registry' => TaskDefinition.new(
destination_path: 'registry.tar.gz',
task: Registry.new(progress)
),
'packages' => TaskDefinition.new(
destination_path: 'packages.tar.gz',
task: Packages.new(progress)
)
}.freeze
end
def create
@tasks.keys.each do |task_name|
@definitions.keys.each do |task_name|
run_create_task(task_name)
end
@ -54,11 +94,11 @@ module Backup
end
def run_create_task(task_name)
task = @tasks[task_name]
definition = @definitions[task_name]
puts_time "Dumping #{task.human_name} ... ".color(:blue)
puts_time "Dumping #{definition.task.human_name} ... ".color(:blue)
unless task.enabled
unless definition.task.enabled
puts_time "[DISABLED]".color(:cyan)
return
end
@ -68,7 +108,8 @@ module Backup
return
end
task.dump
definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path))
puts_time "done".color(:green)
rescue Backup::DatabaseBackupError, Backup::FileBackupError => e
@ -79,39 +120,7 @@ module Backup
cleanup_required = unpack
verify_backup_version
unless skipped?('db')
begin
unless ENV['force'] == 'yes'
warning = <<-MSG.strip_heredoc
Be sure to stop Puma, Sidekiq, and any other process that
connects to the database before proceeding. For Omnibus
installs, see the following link for more information:
https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
Before restoring the database, we will remove all existing
tables to avoid future upgrade problems. Be aware that if you have
custom tables in the GitLab database these tables and all data will be
removed.
MSG
puts warning.color(:red)
Gitlab::TaskHelpers.ask_to_continue
puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
sleep(5)
end
# Drop all tables Load the schema to ensure we don't have any newer tables
# hanging out from a failed upgrade
puts_time 'Cleaning the database ... '.color(:blue)
Rake::Task['gitlab:db:drop_tables'].invoke
puts_time 'done'.color(:green)
run_restore_task('db')
rescue Gitlab::TaskAbortedByUserError
puts "Quitting...".color(:red)
exit 1
end
end
@tasks.except('db').keys.each do |task_name|
@definitions.keys.each do |task_name|
run_restore_task(task_name) unless skipped?(task_name)
end
@ -130,25 +139,44 @@ module Backup
end
def run_restore_task(task_name)
task = @tasks[task_name]
definition = @definitions[task_name]
puts_time "Restoring #{task.human_name} ... ".color(:blue)
puts_time "Restoring #{definition.task.human_name} ... ".color(:blue)
unless task.enabled
unless definition.task.enabled
puts_time "[DISABLED]".color(:cyan)
return
end
task.restore
warning = definition.task.pre_restore_warning
if warning.present?
puts_time warning.color(:red)
Gitlab::TaskHelpers.ask_to_continue
end
definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path))
puts_time "done".color(:green)
warning = definition.task.post_restore_warning
if warning.present?
puts_time warning.color(:red)
Gitlab::TaskHelpers.ask_to_continue
end
rescue Gitlab::TaskAbortedByUserError
puts_time "Quitting...".color(:red)
exit 1
end
private
def write_info
# Make sure there is a connection
ActiveRecord::Base.connection.reconnect!
Dir.chdir(backup_path) do
File.open("#{backup_path}/backup_information.yml", "w+") do |file|
File.open("#{backup_path}/#{MANIFEST_NAME}", "w+") do |file|
file << backup_information.to_yaml.gsub(/^---\n/, '')
end
end
@ -182,8 +210,11 @@ module Backup
upload = directory.files.create(create_attributes)
if upload
progress.puts "done".color(:green)
upload
if upload.respond_to?(:encryption) && upload.encryption
progress.puts "done (encrypted with #{upload.encryption})".color(:green)
else
progress.puts "done".color(:green)
end
else
puts "uploading backup to #{remote_directory} failed".color(:red)
raise Backup::Error, 'Backup failed'
@ -193,18 +224,19 @@ module Backup
def cleanup
progress.print "Deleting tmp directories ... "
backup_contents.each do |dir|
next unless File.exist?(File.join(backup_path, dir))
if FileUtils.rm_rf(File.join(backup_path, dir))
progress.puts "done".color(:green)
else
puts "deleting tmp directory '#{dir}' failed".color(:red)
raise Backup::Error, 'Backup failed'
end
remove_backup_path(MANIFEST_NAME)
@definitions.each do |_, definition|
remove_backup_path(definition.cleanup_path || definition.destination_path)
end
end
def remove_backup_path(path)
return unless File.exist?(File.join(backup_path, path))
FileUtils.rm_rf(File.join(backup_path, path))
progress.puts "done".color(:green)
end
def remove_tmp
# delete tmp inside backups
progress.print "Deleting backups/tmp ... "
@ -322,10 +354,8 @@ module Backup
settings[:skipped] && settings[:skipped].include?(item) || !enabled_task?(item)
end
private
def enabled_task?(task_name)
@tasks[task_name].enabled
@definitions[task_name].task.enabled
end
def backup_file?(file)
@ -333,7 +363,7 @@ module Backup
end
def non_tarred_backup?
File.exist?(File.join(backup_path, 'backup_information.yml'))
File.exist?(File.join(backup_path, MANIFEST_NAME))
end
def backup_path
@ -380,19 +410,14 @@ module Backup
end
def backup_contents
folders_to_backup + archives_to_backup + ["backup_information.yml"]
end
def archives_to_backup
ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact
end
def folders_to_backup
FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) }
[MANIFEST_NAME] + @definitions.reject do |name, definition|
skipped?(name) ||
(definition.destination_optional && !File.exist?(File.join(backup_path, definition.destination_path)))
end.values.map(&:destination_path)
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
@settings ||= YAML.load_file(MANIFEST_NAME)
end
def tar_file

View File

@ -2,14 +2,11 @@
module Backup
class Packages < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('packages', Settings.packages.storage_path, excludes: ['tmp'])
super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp'])
end
override :human_name
def human_name
_('packages')
end

View File

@ -6,14 +6,11 @@ module Backup
# if some of these files are still there, we don't need them in the backup
LEGACY_PAGES_TMP_PATH = '@pages.tmp'
attr_reader :progress
def initialize(progress)
@progress = progress
super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
end
override :human_name
def human_name
_('pages')
end

View File

@ -2,18 +2,16 @@
module Backup
class Registry < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('registry', Settings.registry.path)
super(progress, 'registry', Settings.registry.path)
end
override :human_name
def human_name
_('container registry images')
end
override :enabled
def enabled
Gitlab.config.registry.enabled
end

View File

@ -3,16 +3,20 @@
require 'yaml'
module Backup
class Repositories
class Repositories < Task
extend ::Gitlab::Utils::Override
def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1)
@progress = progress
super(progress)
@strategy = strategy
@max_concurrency = max_concurrency
@max_storage_concurrency = max_storage_concurrency
end
def dump
strategy.start(:create)
override :dump
def dump(path)
strategy.start(:create, path)
# gitaly-backup is designed to handle concurrency on its own. So we want
# to avoid entering the buggy concurrency code here when gitaly-backup
@ -50,8 +54,9 @@ module Backup
strategy.finish!
end
def restore
strategy.start(:restore)
override :restore
def restore(path)
strategy.start(:restore, path)
enqueue_consecutive
ensure
@ -61,17 +66,14 @@ module Backup
restore_object_pools
end
def enabled
true
end
override :human_name
def human_name
_('repositories')
end
private
attr_reader :progress, :strategy, :max_concurrency, :max_storage_concurrency
attr_reader :strategy, :max_concurrency, :max_storage_concurrency
def check_valid_storages!
repository_storage_klasses.each do |klass|

46
lib/backup/task.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Backup
class Task
def initialize(progress)
@progress = progress
end
# human readable task name used for logging
def human_name
raise NotImplementedError
end
# dump task backup to `path`
def dump(path)
raise NotImplementedError
end
# restore task backup from `path`
def restore(path)
raise NotImplementedError
end
# a string returned here will be displayed to the user before calling #restore
def pre_restore_warning
end
# a string returned here will be displayed to the user after calling #restore
def post_restore_warning
end
# returns `true` when the task should be used
def enabled
true
end
private
attr_reader :progress
def puts_time(msg)
progress.puts "#{Time.zone.now} -- #{msg}"
Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
end
end
end

View File

@ -2,14 +2,11 @@
module Backup
class TerraformState < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
end
override :human_name
def human_name
_('terraform states')
end

View File

@ -2,14 +2,11 @@
module Backup
class Uploads < Backup::Files
attr_reader :progress
def initialize(progress)
@progress = progress
super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
end
override :human_name
def human_name
_('uploads')
end

View File

@ -1,32 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'rake'
host = ARGV.shift
ENV['CONTRACT_HOST'] ||= host
list = []
loop do
keyword = ARGV.shift
case keyword
when '--mr'
ENV['CONTRACT_MR'] ||= ARGV.shift
list.push 'test:merge_request'
else
break
end
end
app = Rake.application
Dir.chdir('contracts/provider') do
app.init
app.add_import 'Rakefile'
app.load_rakefile
list.each do |element|
app[element].invoke
end
end

View File

@ -1,17 +0,0 @@
# frozen_string_literal: true
require 'pact/tasks/verification_task'
Pact::VerificationTask.new(:metadata) do |pact|
pact.uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json', pact_helper: './spec/metadata_helper.rb'
end
Pact::VerificationTask.new(:discussions) do |pact|
pact.uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json', pact_helper: './spec/discussions_helper.rb'
end
Pact::VerificationTask.new(:diffs) do |pact|
pact.uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json', pact_helper: './spec/diffs_helper.rb'
end
task 'test:merge_request' => ['pact:verify:metadata', 'pact:verify:discussions', 'pact:verify:diffs']

View File

@ -1,25 +1,23 @@
# frozen_string_literal: true
require 'faraday'
module Provider
module Environments
class Base
attr_writer :base_url, :merge_request
module Environments
class Base
attr_writer :base_url, :merge_request
def call(env)
@payload
end
def call(env)
@payload
end
def http(endpoint)
Faraday.default_adapter = :net_http
response = Faraday.get(@base_url + endpoint)
@payload = [response.status, response.headers, [response.body]]
self
end
def http(endpoint)
Faraday.default_adapter = :net_http
response = Faraday.get(@base_url + endpoint)
@payload = [response.status, response.headers, [response.body]]
self
end
def merge_request(endpoint)
if endpoint.include? '.json'
http(@merge_request + endpoint)
def merge_request(endpoint)
http(@merge_request + endpoint) if endpoint.include? '.json'
end
end
end

View File

@ -1,12 +1,12 @@
# frozen_string_literal: true
require_relative './base'
module Environments
class Local < Base
def initialize
@base_url = ENV['CONTRACT_HOST']
@merge_request = ENV['CONTRACT_MR']
module Provider
module Environments
class Local < Base
def initialize
@base_url = ENV['CONTRACT_HOST']
@merge_request = ENV['CONTRACT_MR']
end
end
end
end

View File

@ -1,15 +1,17 @@
# frozen_string_literal: true
require_relative '../environments/local'
require_relative '../spec_helper'
module DiffsHelper
local = Environments::Local.new
module Provider
module DiffsHelper
local = Environments::Local.new
Pact.service_provider "Merge Request Diffs Endpoint" do
app { local.merge_request('/diffs_batch.json?page=0') }
Pact.service_provider "Merge Request Diffs Endpoint" do
app { local.merge_request('/diffs_batch.json?page=0') }
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json'
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json'
end
end
end
end

View File

@ -1,15 +1,17 @@
# frozen_string_literal: true
require_relative '../environments/local'
require_relative '../spec_helper'
module DiscussionsHelper
local = Environments::Local.new
module Provider
module DiscussionsHelper
local = Environments::Local.new
Pact.service_provider "Merge Request Discussions Endpoint" do
app { local.merge_request('/discussions.json') }
Pact.service_provider "Merge Request Discussions Endpoint" do
app { local.merge_request('/discussions.json') }
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json'
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json'
end
end
end
end

View File

@ -1,15 +1,17 @@
# frozen_string_literal: true
require_relative '../environments/local'
require_relative '../spec_helper'
module MetadataHelper
local = Environments::Local.new
module Provider
module MetadataHelper
local = Environments::Local.new
Pact.service_provider "Merge Request Metadata Endpoint" do
app { local.merge_request('/diffs_metadata.json') }
Pact.service_provider "Merge Request Metadata Endpoint" do
app { local.merge_request('/diffs_metadata.json') }
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json'
honours_pact_with 'Merge Request Page' do
pact_uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json'
end
end
end
end

View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
module SpecHelper
unless ENV['CONTRACT_HOST']
raise(ArgumentError, 'Contract tests require CONTRACT_HOST environment variable to be set!')
end
require_relative '../../../config/bundler_setup'
Bundler.require(:default)
root = File.expand_path('../', __dir__)
loader = Zeitwerk::Loader.new
loader.push_dir(root)
loader.ignore("#{root}/consumer")
loader.ignore("#{root}/contracts")
loader.collapse("#{root}/provider/spec")
loader.setup
end

47
qa/tasks/contracts.rake Normal file
View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
require 'pact/tasks/verification_task'
contracts = File.expand_path('../contracts', __dir__)
provider = File.expand_path('provider', contracts)
# rubocop:disable Rails/RakeEnvironment
namespace :contracts do
namespace :mr do
Pact::VerificationTask.new(:metadata) do |pact|
pact.uri(
"#{contracts}/contracts/merge_request_page-merge_request_metadata_endpoint.json",
pact_helper: "#{provider}/spec/metadata_helper.rb"
)
end
Pact::VerificationTask.new(:discussions) do |pact|
pact.uri(
"#{contracts}/contracts/merge_request_page-merge_request_discussions_endpoint.json",
pact_helper: "#{provider}/spec/discussions_helper.rb"
)
end
Pact::VerificationTask.new(:diffs) do |pact|
pact.uri(
"#{contracts}/contracts/merge_request_page-merge_request_diffs_endpoint.json",
pact_helper: "#{provider}/spec/diffs_helper.rb"
)
end
desc 'Run all merge request contract tests'
task 'test:merge_request', :contract_mr do |_t, arg|
raise(ArgumentError, 'Merge request contract tests require contract_mr to be set') unless arg[:contract_mr]
ENV['CONTRACT_MR'] = arg[:contract_mr]
errors = %w[metadata discussions diffs].each_with_object([]) do |task, err|
Rake::Task["contracts:mr:pact:verify:#{task}"].execute
rescue StandardError, SystemExit
err << "contracts:mr:pact:verify:#{task}"
end
raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty?
end
end
end
# rubocop:enable Rails/RakeEnvironment

View File

@ -41,7 +41,8 @@ export const getSecurityTrainingProvidersData = (providerOverrides = {}) => {
const response = {
data: {
project: {
id: 1,
id: 'gid://gitlab/Project/1',
__typename: 'Project',
securityTrainingProviders,
},
},

View File

@ -18,7 +18,7 @@ RSpec.describe Backup::Artifacts do
expect(backup).to receive(:tar).and_return('blabla-tar')
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump
backup.dump('artifacts.tar.gz')
end
end
end

View File

@ -6,25 +6,49 @@ RSpec.describe Backup::Database do
let(:progress) { StringIO.new }
let(:output) { progress.string }
before do
allow(Gitlab::TaskHelpers).to receive(:ask_to_continue)
before(:all) do
Rake.application.rake_require 'active_record/railties/databases'
Rake.application.rake_require 'tasks/gitlab/backup'
Rake.application.rake_require 'tasks/gitlab/shell'
Rake.application.rake_require 'tasks/gitlab/db'
Rake.application.rake_require 'tasks/cache'
end
describe '#restore' do
let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
let(:force) { true }
subject { described_class.new(progress, filename: data) }
subject { described_class.new(progress, force: force) }
before do
allow(subject).to receive(:pg_restore_cmd).and_return(cmd)
end
context 'when not forced' do
let(:force) { false }
it 'warns the user and waits' do
expect(subject).to receive(:sleep)
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include('Removing all tables. Press `Ctrl-C` within 5 seconds to abort')
end
it 'has a pre restore warning' do
expect(subject.pre_restore_warning).not_to be_nil
end
end
context 'with an empty .gz file' do
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
it 'returns successfully' do
subject.restore
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include("Restoring PostgreSQL database")
expect(output).to include("[DONE]")
@ -36,7 +60,9 @@ RSpec.describe Backup::Database do
let(:data) { Rails.root.join("spec/fixtures/big-image.png").to_s }
it 'raises a backup error' do
expect { subject.restore }.to raise_error(Backup::Error)
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect { subject.restore(data) }.to raise_error(Backup::Error)
end
end
@ -45,12 +71,15 @@ RSpec.describe Backup::Database do
let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\nWARNING: no privileges could be revoked for public\n" }
let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
it 'filters out noise from errors' do
subject.restore
it 'filters out noise from errors and has a post restore warning' do
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include("ERRORS")
expect(output).not_to include(noise)
expect(output).to include(visible_error)
expect(subject.post_restore_warning).not_to be_nil
end
end
@ -66,7 +95,9 @@ RSpec.describe Backup::Database do
end
it 'overrides default config values' do
subject.restore
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include(%("PGHOST"=>"test.example.com"))
expect(output).to include(%("PGPASSWORD"=>"donotchange"))

View File

@ -39,7 +39,7 @@ RSpec.describe Backup::Files do
end
describe '#restore' do
subject { described_class.new('registry', '/var/gitlab-registry') }
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
let(:timestamp) { Time.utc(2017, 3, 22) }
@ -58,11 +58,11 @@ RSpec.describe Backup::Files do
it 'moves all necessary files' do
allow(subject).to receive(:backup_existing_files).and_call_original
expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}"))
subject.restore
subject.restore('registry.tar.gz')
end
it 'raises no errors' do
expect { subject.restore }.not_to raise_error
expect { subject.restore('registry.tar.gz') }.not_to raise_error
end
it 'calls tar command with unlink' do
@ -70,13 +70,13 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
expect(subject).to receive(:pipeline_succeeded?).and_return(true)
subject.restore
subject.restore('registry.tar.gz')
end
it 'raises an error on failure' do
expect(subject).to receive(:pipeline_succeeded?).and_return(false)
expect { subject.restore }.to raise_error(/Restore operation failed:/)
expect { subject.restore('registry.tar.gz') }.to raise_error(/Restore operation failed:/)
end
end
@ -89,7 +89,7 @@ RSpec.describe Backup::Files do
it 'shows error message' do
expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry")
subject.restore
subject.restore('registry.tar.gz')
end
end
@ -104,13 +104,13 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:resource_busy_error).with("/var/gitlab-registry")
.and_call_original
expect { subject.restore }.to raise_error(/is a mountpoint/)
expect { subject.restore('registry.tar.gz') }.to raise_error(/is a mountpoint/)
end
end
end
describe '#dump' do
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
before do
allow(subject).to receive(:run_pipeline!).and_return([[true, true], ''])
@ -118,14 +118,14 @@ RSpec.describe Backup::Files do
end
it 'raises no errors' do
expect { subject.dump }.not_to raise_error
expect { subject.dump('registry.tar.gz') }.not_to raise_error
end
it 'excludes tmp dirs from archive' do
expect(subject).to receive(:tar).and_return('blabla-tar')
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args)
subject.dump
subject.dump('registry.tar.gz')
end
it 'raises an error on failure' do
@ -133,7 +133,7 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:pipeline_succeeded?).and_return(false)
expect do
subject.dump
subject.dump('registry.tar.gz')
end.to raise_error(/Failed to create compressed file/)
end
@ -149,7 +149,7 @@ RSpec.describe Backup::Files do
.with(%w(rsync -a --delete --exclude=lost+found --exclude=/gitlab-pages/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
.and_return(['', 0])
subject.dump
subject.dump('registry.tar.gz')
end
it 'retries if rsync fails due to vanishing files' do
@ -158,7 +158,7 @@ RSpec.describe Backup::Files do
.and_return(['rsync failed', 24], ['', 0])
expect do
subject.dump
subject.dump('registry.tar.gz')
end.to output(/files vanished during rsync, retrying/).to_stdout
end
@ -168,7 +168,7 @@ RSpec.describe Backup::Files do
.and_return(['rsync failed', 1])
expect do
subject.dump
subject.dump('registry.tar.gz')
end.to output(/rsync failed/).to_stdout
.and raise_error(/Failed to create compressed file/)
end
@ -176,7 +176,7 @@ RSpec.describe Backup::Files do
end
describe '#exclude_dirs' do
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
it 'prepends a leading dot slash to tar excludes' do
expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp'])
@ -188,7 +188,7 @@ RSpec.describe Backup::Files do
end
describe '#run_pipeline!' do
subject { described_class.new('registry', '/var/gitlab-registry') }
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'executes an Open3.pipeline for cmd_list' do
expect(Open3).to receive(:pipeline).with(%w[whew command], %w[another cmd], any_args)
@ -222,7 +222,7 @@ RSpec.describe Backup::Files do
end
describe '#pipeline_succeeded?' do
subject { described_class.new('registry', '/var/gitlab-registry') }
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'returns true if both tar and gzip succeeeded' do
expect(
@ -262,7 +262,7 @@ RSpec.describe Backup::Files do
end
describe '#tar_ignore_non_success?' do
subject { described_class.new('registry', '/var/gitlab-registry') }
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
context 'if `tar` command exits with 1 exitstatus' do
it 'returns true' do
@ -310,7 +310,7 @@ RSpec.describe Backup::Files do
end
describe '#noncritical_warning?' do
subject { described_class.new('registry', '/var/gitlab-registry') }
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'returns true if given text matches noncritical warnings list' do
expect(

View File

@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe Backup::GitalyBackup do
let(:max_parallelism) { nil }
let(:storage_parallelism) { nil }
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
let(:progress) do
Tempfile.new('progress').tap do |progress|
@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyBackup do
context 'unknown' do
it 'fails to start unknown' do
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
end
end
@ -42,7 +43,7 @@ RSpec.describe Backup::GitalyBackup do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original
subject.start(:create)
subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
@ -50,11 +51,11 @@ RSpec.describe Backup::GitalyBackup do
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.finish!
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
end
context 'parallel option set' do
@ -63,7 +64,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original
subject.start(:create)
subject.start(:create, destination)
subject.finish!
end
end
@ -74,7 +75,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original
subject.start(:create)
subject.start(:create, destination)
subject.finish!
end
end
@ -82,7 +83,7 @@ RSpec.describe Backup::GitalyBackup do
it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:create)
subject.start(:create, destination)
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end
end
@ -114,7 +115,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes through SSL envs' do
expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original
subject.start(:create)
subject.start(:create, destination)
subject.finish!
end
end
@ -139,7 +140,7 @@ RSpec.describe Backup::GitalyBackup do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original
subject.start(:restore)
subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
@ -162,7 +163,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original
subject.start(:restore)
subject.start(:restore, destination)
subject.finish!
end
end
@ -173,7 +174,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original
subject.start(:restore)
subject.start(:restore, destination)
subject.finish!
end
end
@ -181,7 +182,7 @@ RSpec.describe Backup::GitalyBackup do
it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:restore)
subject.start(:restore, destination)
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end
end

View File

@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe Backup::GitalyRpcBackup do
let(:progress) { spy(:stdout) }
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
subject { described_class.new(progress) }
@ -14,7 +15,7 @@ RSpec.describe Backup::GitalyRpcBackup do
context 'unknown' do
it 'fails to start unknown' do
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
end
end
@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyRpcBackup do
project_snippet = create(:project_snippet, :repository, project: project)
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
subject.start(:create)
subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
@ -35,11 +36,11 @@ RSpec.describe Backup::GitalyRpcBackup do
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.finish!
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
end
context 'failure' do
@ -50,7 +51,7 @@ RSpec.describe Backup::GitalyRpcBackup do
end
it 'logs an appropriate message', :aggregate_failures do
subject.start(:create)
subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.finish!
@ -90,7 +91,7 @@ RSpec.describe Backup::GitalyRpcBackup do
copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
subject.start(:restore)
subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
@ -123,7 +124,7 @@ RSpec.describe Backup::GitalyRpcBackup do
repository
end
subject.start(:restore)
subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
@ -141,7 +142,7 @@ RSpec.describe Backup::GitalyRpcBackup do
end
it 'logs an appropriate message', :aggregate_failures do
subject.start(:restore)
subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.finish!

View File

@ -20,7 +20,7 @@ RSpec.describe Backup::Lfs do
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump
backup.dump('lfs.tar.gz')
end
end
end

File diff suppressed because it is too large Load Diff

View File

@ -21,7 +21,7 @@ RSpec.shared_examples 'backup object' do |setting|
expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump
backup.dump('backup_object.tar.gz')
end
end
end

View File

@ -19,7 +19,7 @@ RSpec.describe Backup::Pages do
expect(subject).to receive(:tar).and_return('blabla-tar')
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(subject).to receive(:pipeline_succeeded?).and_return(true)
subject.dump
subject.dump('pages.tar.gz')
end
end
end

View File

@ -8,6 +8,7 @@ RSpec.describe Backup::Repositories do
let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
let(:max_concurrency) { 1 }
let(:max_storage_concurrency) { 1 }
let(:destination) { 'repositories' }
subject do
described_class.new(
@ -26,9 +27,9 @@ RSpec.describe Backup::Repositories do
project_snippet = create(:project_snippet, :repository, project: project)
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
subject.dump
subject.dump(destination)
expect(strategy).to have_received(:start).with(:create)
expect(strategy).to have_received(:start).with(:create, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
@ -54,38 +55,38 @@ RSpec.describe Backup::Repositories do
it 'creates the expected number of threads' do
expect(Thread).not_to receive(:new)
expect(strategy).to receive(:start).with(:create)
expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end
expect(strategy).to receive(:finish!)
subject.dump
subject.dump(destination)
end
describe 'command failure' do
it 'enqueue_project raises an error' do
allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
expect { subject.dump }.to raise_error(IOError)
expect { subject.dump(destination) }.to raise_error(IOError)
end
it 'project query raises an error' do
allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
end
end
it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new do
subject.dump
subject.dump(destination)
end.count
create_list(:project, 2, :repository)
expect do
subject.dump
subject.dump(destination)
end.not_to exceed_query_limit(control_count)
end
end
@ -98,13 +99,13 @@ RSpec.describe Backup::Repositories do
it 'enqueues all projects sequentially' do
expect(Thread).not_to receive(:new)
expect(strategy).to receive(:start).with(:create)
expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end
expect(strategy).to receive(:finish!)
subject.dump
subject.dump(destination)
end
end
@ -122,13 +123,13 @@ RSpec.describe Backup::Repositories do
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times
.and_call_original
expect(strategy).to receive(:start).with(:create)
expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end
expect(strategy).to receive(:finish!)
subject.dump
subject.dump(destination)
end
context 'with extra max concurrency' do
@ -139,13 +140,13 @@ RSpec.describe Backup::Repositories do
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times
.and_call_original
expect(strategy).to receive(:start).with(:create)
expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end
expect(strategy).to receive(:finish!)
subject.dump
subject.dump(destination)
end
end
@ -153,33 +154,33 @@ RSpec.describe Backup::Repositories do
it 'enqueue_project raises an error' do
allow(strategy).to receive(:enqueue).and_raise(IOError)
expect { subject.dump }.to raise_error(IOError)
expect { subject.dump(destination) }.to raise_error(IOError)
end
it 'project query raises an error' do
allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
end
context 'misconfigured storages' do
let(:storage_keys) { %w[test_second_storage] }
it 'raises an error' do
expect { subject.dump }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
expect { subject.dump(destination) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
end
end
end
it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new do
subject.dump
subject.dump(destination)
end.count
create_list(:project, 2, :repository)
expect do
subject.dump
subject.dump(destination)
end.not_to exceed_query_limit(control_count)
end
end
@ -192,9 +193,9 @@ RSpec.describe Backup::Repositories do
let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.first_owner) }
it 'calls enqueue for each repository type', :aggregate_failures do
subject.restore
subject.restore(destination)
expect(strategy).to have_received(:start).with(:restore)
expect(strategy).to have_received(:start).with(:restore, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
@ -208,7 +209,7 @@ RSpec.describe Backup::Repositories do
pool_repository = create(:pool_repository, :failed)
pool_repository.delete_object_pool
subject.restore
subject.restore(destination)
pool_repository.reload
expect(pool_repository).not_to be_failed
@ -219,7 +220,7 @@ RSpec.describe Backup::Repositories do
pool_repository = create(:pool_repository, state: :obsolete)
pool_repository.update_column(:source_project_id, nil)
subject.restore
subject.restore(destination)
pool_repository.reload
expect(pool_repository).to be_obsolete
@ -236,14 +237,14 @@ RSpec.describe Backup::Repositories do
end
it 'shows the appropriate error' do
subject.restore
subject.restore(destination)
expect(progress).to have_received(:puts).with("Snippet #{personal_snippet.full_path} can't be restored: Repository has more than one branch")
expect(progress).to have_received(:puts).with("Snippet #{project_snippet.full_path} can't be restored: Repository has more than one branch")
end
it 'removes the snippets from the DB' do
expect { subject.restore }.to change(PersonalSnippet, :count).by(-1)
expect { subject.restore(destination) }.to change(PersonalSnippet, :count).by(-1)
.and change(ProjectSnippet, :count).by(-1)
.and change(SnippetRepository, :count).by(-2)
end
@ -253,7 +254,7 @@ RSpec.describe Backup::Repositories do
shard_name = personal_snippet.repository.shard
path = personal_snippet.disk_path + '.git'
subject.restore
subject.restore(destination)
expect(gitlab_shell.repository_exists?(shard_name, path)).to eq false
end

View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Backup::Task do
let(:progress) { StringIO.new }
subject { described_class.new(progress) }
describe '#human_name' do
it 'must be implemented by the subclass' do
expect { subject.human_name }.to raise_error(NotImplementedError)
end
end
describe '#dump' do
it 'must be implemented by the subclass' do
expect { subject.dump('some/path') }.to raise_error(NotImplementedError)
end
end
describe '#restore' do
it 'must be implemented by the subclass' do
expect { subject.restore('some/path') }.to raise_error(NotImplementedError)
end
end
end

View File

@ -19,7 +19,7 @@ RSpec.describe Backup::Uploads do
expect(backup).to receive(:tar).and_return('blabla-tar')
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump
backup.dump('uploads.tar.gz')
end
end
end

View File

@ -72,7 +72,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
before do
allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: gitlab_version })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
@ -85,10 +84,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
it 'invokes restoration on match' do
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
end
it 'prints timestamps on messages' do
expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout_from_any_process
end
end
end
@ -131,8 +126,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: Gitlab::VERSION })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
@ -486,7 +479,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect_next_instance_of(::Backup::Manager) do |instance|
(backup_types - %w{repositories uploads}).each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered
@ -531,7 +523,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered