Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
34e729c8b9
commit
c16d7a4f01
|
|
@ -0,0 +1 @@
|
|||
<svg enable-background="new 0 0 50 50" viewBox="0 0 50 50" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><linearGradient id="a" gradientUnits="userSpaceOnUse" x1="10.2941" x2="50" y1="17.8247" y2="17.8247"><stop offset="0" stop-color="#f39c63"/><stop offset="1" stop-color="#ef5d4f"/></linearGradient><linearGradient id="b"><stop offset="0" stop-color="#231c4f"/><stop offset="1" stop-color="#0a0430"/></linearGradient><linearGradient id="c" gradientUnits="userSpaceOnUse" x1="16.9118" x2="25" xlink:href="#b" y1="27.2046" y2="27.2046"/><linearGradient id="d" gradientUnits="userSpaceOnUse" x1="27.9412" x2="33.0882" xlink:href="#b" y1="22.0575" y2="22.0575"/><g clip-rule="evenodd" fill-rule="evenodd"><path d="m31.94 41.71-31.94 8.41v-21.23c0-4.73 3.19-8.87 7.77-10.07l19.29-5.08c6.4-1.68 12.65 3.14 12.65 9.75v8.14c0 4.74-3.19 8.88-7.77 10.08z" fill="#2b6af9"/><path d="m50 0v21.23c0 4.73-3.19 8.87-7.77 10.07l-14.79 3.89c-8.67 2.28-17.15-4.26-17.15-13.22v-3.49c0-4.73 3.19-8.87 7.77-10.07z" fill="url(#a)"/><path d="m23.36 36.33 16.35-4.3v-8.16c0-6.83-6.46-11.81-13.06-10.07l-16.35 4.3v8.16c-.01 6.82 6.45 11.8 13.06 10.07z" fill="#fff"/><circle cx="20.96" cy="27.2" fill="url(#c)" r="4.04"/><circle cx="30.51" cy="22.06" fill="url(#d)" r="2.57"/></g></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
|
|
@ -0,0 +1 @@
|
|||
<svg enable-background="new 0 0 0 0" viewBox="0 0 800 780" xmlns="http://www.w3.org/2000/svg"><path d="m594.4 737.87c-.75-1.93-1.86-3.7-3.34-5.29-1.48-1.6-3.29-2.86-5.44-3.8-2.15-.93-4.62-1.4-7.41-1.4s-5.26.47-7.41 1.4c-2.14.94-3.96 2.21-5.43 3.8-1.48 1.59-2.59 3.36-3.34 5.29s-1.13 3.91-1.13 5.95v.96c0 1.96.36 3.91 1.1 5.86.73 1.96 1.82 3.74 3.28 5.35s3.27 2.91 5.43 3.89c2.17.98 4.67 1.46 7.5 1.46s5.33-.49 7.5-1.46c2.16-.98 3.98-2.28 5.43-3.89 1.46-1.61 2.55-3.4 3.28-5.35s1.1-3.91 1.1-5.86v-.96c.01-2.04-.37-4.02-1.12-5.95zm-4.01 13.45c-1.15 2.1-2.78 3.77-4.86 5.02-2.09 1.25-4.52 1.89-7.32 1.89-2.79 0-5.24-.63-7.32-1.89-2.09-1.25-3.71-2.93-4.86-5.02s-1.73-4.42-1.73-6.97c0-2.63.58-4.99 1.73-7.09 1.15-2.09 2.77-3.74 4.86-4.96 2.08-1.21 4.52-1.82 7.32-1.82 2.79 0 5.23.61 7.32 1.82 2.08 1.22 3.71 2.87 4.86 4.96 1.15 2.1 1.73 4.46 1.73 7.09 0 2.56-.58 4.88-1.73 6.97z" fill="#f79200"/><path d="m584.11 746.03c1.42-1.12 2.12-2.73 2.12-4.84s-.71-3.73-2.12-4.85c-1.42-1.11-3.4-1.67-5.95-1.67h-2.55-2.37-2.12v18.66h4.49v-5.62h2.55c.09 0 .17-.01.25-.01l3.81 5.64h5.1l-4.69-6.45c.54-.25 1.04-.52 1.48-.86zm-8.5-7.6h2.85c1.1 0 1.9.23 2.43.69s.79 1.15.79 2.07-.26 1.6-.79 2.06-1.33.68-2.43.68h-2.85z" fill="#f79200"/><path d="m333.8 753.41-17.62 7.9v-88.45s76.18-50.47 93.76-64.21c46.25-36.15 101.39-92.54 101.39-155.86v-22.33l40.76-32.62h-235.91v-83.73h280.18v138.69c0 180.72-251.84 295.81-262.56 300.61zm-17.53-638.54c21.22 2.88 121.7 20.25 195.07 97.32v62.85h85.03v-94.02l-10-11.68c-104.42-122.2-259.7-137.72-266.27-138.32l-3.92-.36v84.23c.03-.01.06-.01.09-.02z" fill="#ffbe12"/><path d="m316.18 397.85h-280.18v-216.82l9.99-11.68c104.42-122.21 259.7-137.73 266.25-138.33l3.94-.36v84.23c-21.06 2.97-122.24 20.87-195.17 97.32v74.53l-36.5 27.39h231.66v83.72zm-93.77 210.81c-46.27-36.15-101.4-87.26-101.4-143.31l20.24-28.41h-105.25v28.41c0 171.17 251.84 283.27 262.56 288.07l17.63 7.9v-88.45c-.01 0-76.2-50.47-93.78-64.21z" fill="#f79200"/></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
|
|
@ -284,9 +284,9 @@ export const AUTO_DEVOPS_ENABLED_ALERT_DISMISSED_STORAGE_KEY =
|
|||
// https://gitlab.com/gitlab-org/gitlab/-/issues/346899
|
||||
export const TEMP_PROVIDER_LOGOS = {
|
||||
Kontra: {
|
||||
svg: '/assets/illustrations/vulnerability/kontra-logo.svg',
|
||||
svg: '/assets/vulnerability/kontra-logo.svg',
|
||||
},
|
||||
[__('Secure Code Warrior')]: {
|
||||
svg: '/assets/illustrations/vulnerability/scw-logo.svg',
|
||||
svg: '/assets/vulnerability/scw-logo.svg',
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
query getSecurityTrainingVulnerability($id: ID!) {
|
||||
vulnerability(id: $id) @client {
|
||||
query getSecurityTrainingUrls($projectFullPath: ID!, $identifierExternalIds: [String!]!) {
|
||||
project(fullPath: $projectFullPath) {
|
||||
id
|
||||
identifiers {
|
||||
externalType
|
||||
}
|
||||
securityTrainingUrls {
|
||||
securityTrainingUrls(identifierExternalIds: $identifierExternalIds) {
|
||||
name
|
||||
url
|
||||
status
|
||||
url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
|
|||
push_frontend_feature_flag(:refactor_mr_widgets_extensions, project, default_enabled: :yaml)
|
||||
push_frontend_feature_flag(:rebase_without_ci_ui, project, default_enabled: :yaml)
|
||||
push_frontend_feature_flag(:markdown_continue_lists, project, default_enabled: :yaml)
|
||||
push_frontend_feature_flag(:secure_vulnerability_training, project, default_enabled: :yaml)
|
||||
# Usage data feature flags
|
||||
push_frontend_feature_flag(:users_expanding_widgets_usage_data, project, default_enabled: :yaml)
|
||||
push_frontend_feature_flag(:diff_settings_usage_data, default_enabled: :yaml)
|
||||
|
|
|
|||
|
|
@ -66,20 +66,20 @@ module Users
|
|||
|
||||
# rubocop: disable CodeReuse/ActiveRecord
|
||||
def migrate_issues
|
||||
user.issues.update_all(author_id: ghost_user.id)
|
||||
Issue.where(last_edited_by_id: user.id).update_all(last_edited_by_id: ghost_user.id)
|
||||
batched_migrate(Issue, :author_id)
|
||||
batched_migrate(Issue, :last_edited_by_id)
|
||||
end
|
||||
# rubocop: enable CodeReuse/ActiveRecord
|
||||
|
||||
# rubocop: disable CodeReuse/ActiveRecord
|
||||
def migrate_merge_requests
|
||||
user.merge_requests.update_all(author_id: ghost_user.id)
|
||||
MergeRequest.where(merge_user_id: user.id).update_all(merge_user_id: ghost_user.id)
|
||||
batched_migrate(MergeRequest, :author_id)
|
||||
batched_migrate(MergeRequest, :merge_user_id)
|
||||
end
|
||||
# rubocop: enable CodeReuse/ActiveRecord
|
||||
|
||||
def migrate_notes
|
||||
user.notes.update_all(author_id: ghost_user.id)
|
||||
batched_migrate(Note, :author_id)
|
||||
end
|
||||
|
||||
def migrate_abuse_reports
|
||||
|
|
@ -96,8 +96,17 @@ module Users
|
|||
end
|
||||
|
||||
def migrate_reviews
|
||||
user.reviews.update_all(author_id: ghost_user.id)
|
||||
batched_migrate(Review, :author_id)
|
||||
end
|
||||
|
||||
# rubocop:disable CodeReuse/ActiveRecord
|
||||
def batched_migrate(base_scope, column)
|
||||
loop do
|
||||
update_count = base_scope.where(column => user.id).limit(100).update_all(column => ghost_user.id)
|
||||
break if update_count == 0
|
||||
end
|
||||
end
|
||||
# rubocop:enable CodeReuse/ActiveRecord
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -749,6 +749,38 @@ deploystacks:
|
|||
- ${PROVIDER}-${STACK}
|
||||
```
|
||||
|
||||
#### Fetch artifacts from a `parallel:matrix` job
|
||||
|
||||
You can fetch artifacts from a job created with [`parallel:matrix`](../yaml/index.md#parallelmatrix)
|
||||
by using the [`dependencies`](../yaml/index.md#dependencies) keyword. Use the job name
|
||||
as the value for `dependencies` as a string in the form:
|
||||
|
||||
```plaintext
|
||||
<job_name> [<matrix argument 1>, <matrix argument 2>, ... <matrix argument N>]
|
||||
```
|
||||
|
||||
For example, to fetch the artifacts from the job with a `RUBY_VERSION` of `2.7` and
|
||||
a `PROVIDER` of `aws`:
|
||||
|
||||
```yaml
|
||||
ruby:
|
||||
image: ruby:${RUBY_VERSION}
|
||||
parallel:
|
||||
matrix:
|
||||
- RUBY_VERSION: ["2.5", "2.6", "2.7", "3.0", "3.1"]
|
||||
PROVIDER: [aws, gcp]
|
||||
script: bundle install
|
||||
|
||||
deploy:
|
||||
image: ruby:2.7
|
||||
stage: deploy
|
||||
dependencies:
|
||||
- "ruby: [2.7, aws]"
|
||||
script: echo hello
|
||||
```
|
||||
|
||||
Quotes around the `dependencies` entry are required.
|
||||
|
||||
## Use predefined CI/CD variables to run jobs only in specific pipeline types
|
||||
|
||||
You can use [predefined CI/CD variables](../variables/predefined_variables.md) to choose
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ results. On failure, the analyzer outputs an
|
|||
## Prerequisites
|
||||
|
||||
- [GitLab Runner](../../../ci/runners/index.md) available, with the
|
||||
[`docker` executor](https://docs.gitlab.com/runner/executors/docker.html).
|
||||
[`docker` executor](https://docs.gitlab.com/runner/executors/docker.html) on Linux/amd64.
|
||||
- Target application deployed. For more details, read [Deployment options](#deployment-options).
|
||||
- DAST runs in the `dast` stage, which must be added manually to your `.gitlab-ci.yml`.
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ To run SAST jobs, by default, you need GitLab Runner with the
|
|||
If you're using the shared runners on GitLab.com, this is enabled by default.
|
||||
|
||||
WARNING:
|
||||
Our SAST jobs require a Linux container type. Windows containers are not yet supported.
|
||||
Our SAST jobs require a Linux/amd64 container type. Windows containers are not yet supported.
|
||||
|
||||
WARNING:
|
||||
If you use your own runners, make sure the Docker version installed
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class Artifacts < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('artifacts', JobArtifactUploader.root, excludes: ['tmp'])
|
||||
super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp'])
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('artifacts')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class Builds < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('builds', Settings.gitlab_ci.builds_path)
|
||||
super(progress, 'builds', Settings.gitlab_ci.builds_path)
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('builds')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@
|
|||
require 'yaml'
|
||||
|
||||
module Backup
|
||||
class Database
|
||||
class Database < Task
|
||||
extend ::Gitlab::Utils::Override
|
||||
include Backup::Helper
|
||||
attr_reader :progress
|
||||
attr_reader :config, :db_file_name
|
||||
attr_reader :force, :config
|
||||
|
||||
IGNORED_ERRORS = [
|
||||
# Ignore warnings
|
||||
|
|
@ -18,13 +18,14 @@ module Backup
|
|||
].freeze
|
||||
IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze
|
||||
|
||||
def initialize(progress, filename: nil)
|
||||
@progress = progress
|
||||
def initialize(progress, force:)
|
||||
super(progress)
|
||||
@config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
|
||||
@db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
|
||||
@force = force
|
||||
end
|
||||
|
||||
def dump
|
||||
override :dump
|
||||
def dump(db_file_name)
|
||||
FileUtils.mkdir_p(File.dirname(db_file_name))
|
||||
FileUtils.rm_f(db_file_name)
|
||||
compress_rd, compress_wr = IO.pipe
|
||||
|
|
@ -64,12 +65,24 @@ module Backup
|
|||
raise DatabaseBackupError.new(config, db_file_name) unless success
|
||||
end
|
||||
|
||||
def restore
|
||||
override :restore
|
||||
def restore(db_file_name)
|
||||
unless force
|
||||
progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
|
||||
sleep(5)
|
||||
end
|
||||
|
||||
# Drop all tables Load the schema to ensure we don't have any newer tables
|
||||
# hanging out from a failed upgrade
|
||||
puts_time 'Cleaning the database ... '.color(:blue)
|
||||
Rake::Task['gitlab:db:drop_tables'].invoke
|
||||
puts_time 'done'.color(:green)
|
||||
|
||||
decompress_rd, decompress_wr = IO.pipe
|
||||
decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
|
||||
decompress_wr.close
|
||||
|
||||
status, errors =
|
||||
status, @errors =
|
||||
case config[:adapter]
|
||||
when "postgresql" then
|
||||
progress.print "Restoring PostgreSQL database #{database} ... "
|
||||
|
|
@ -81,33 +94,47 @@ module Backup
|
|||
Process.waitpid(decompress_pid)
|
||||
success = $?.success? && status.success?
|
||||
|
||||
if errors.present?
|
||||
if @errors.present?
|
||||
progress.print "------ BEGIN ERRORS -----\n".color(:yellow)
|
||||
progress.print errors.join.color(:yellow)
|
||||
progress.print @errors.join.color(:yellow)
|
||||
progress.print "------ END ERRORS -------\n".color(:yellow)
|
||||
end
|
||||
|
||||
report_success(success)
|
||||
raise Backup::Error, 'Restore failed' unless success
|
||||
|
||||
if errors.present?
|
||||
warning = <<~MSG
|
||||
There were errors in restoring the schema. This may cause
|
||||
issues if this results in missing indexes, constraints, or
|
||||
columns. Please record the errors above and contact GitLab
|
||||
Support if you have questions:
|
||||
https://about.gitlab.com/support/
|
||||
MSG
|
||||
|
||||
warn warning.color(:red)
|
||||
Gitlab::TaskHelpers.ask_to_continue
|
||||
end
|
||||
end
|
||||
|
||||
def enabled
|
||||
true
|
||||
override :pre_restore_warning
|
||||
def pre_restore_warning
|
||||
return if force
|
||||
|
||||
<<-MSG.strip_heredoc
|
||||
Be sure to stop Puma, Sidekiq, and any other process that
|
||||
connects to the database before proceeding. For Omnibus
|
||||
installs, see the following link for more information:
|
||||
https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
|
||||
|
||||
Before restoring the database, we will remove all existing
|
||||
tables to avoid future upgrade problems. Be aware that if you have
|
||||
custom tables in the GitLab database these tables and all data will be
|
||||
removed.
|
||||
MSG
|
||||
end
|
||||
|
||||
override :post_restore_warning
|
||||
def post_restore_warning
|
||||
return unless @errors.present?
|
||||
|
||||
<<-MSG.strip_heredoc
|
||||
There were errors in restoring the schema. This may cause
|
||||
issues if this results in missing indexes, constraints, or
|
||||
columns. Please record the errors above and contact GitLab
|
||||
Support if you have questions:
|
||||
https://about.gitlab.com/support/
|
||||
MSG
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('database')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,25 +1,27 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'open3'
|
||||
require_relative 'helper'
|
||||
|
||||
module Backup
|
||||
class Files
|
||||
class Files < Task
|
||||
extend ::Gitlab::Utils::Override
|
||||
include Backup::Helper
|
||||
|
||||
DEFAULT_EXCLUDE = 'lost+found'
|
||||
|
||||
attr_reader :name, :backup_tarball, :excludes
|
||||
attr_reader :name, :excludes
|
||||
|
||||
def initialize(progress, name, app_files_dir, excludes: [])
|
||||
super(progress)
|
||||
|
||||
def initialize(name, app_files_dir, excludes: [])
|
||||
@name = name
|
||||
@app_files_dir = app_files_dir
|
||||
@backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
|
||||
@excludes = [DEFAULT_EXCLUDE].concat(excludes)
|
||||
end
|
||||
|
||||
# Copy files from public/files to backup/files
|
||||
def dump
|
||||
override :dump
|
||||
def dump(backup_tarball)
|
||||
FileUtils.mkdir_p(Gitlab.config.backup.path)
|
||||
FileUtils.rm_f(backup_tarball)
|
||||
|
||||
|
|
@ -35,7 +37,7 @@ module Backup
|
|||
|
||||
unless status == 0
|
||||
puts output
|
||||
raise_custom_error
|
||||
raise_custom_error(backup_tarball)
|
||||
end
|
||||
|
||||
tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten
|
||||
|
|
@ -47,11 +49,12 @@ module Backup
|
|||
end
|
||||
|
||||
unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output)
|
||||
raise_custom_error
|
||||
raise_custom_error(backup_tarball)
|
||||
end
|
||||
end
|
||||
|
||||
def restore
|
||||
override :restore
|
||||
def restore(backup_tarball)
|
||||
backup_existing_files_dir
|
||||
|
||||
cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]]
|
||||
|
|
@ -61,10 +64,6 @@ module Backup
|
|||
end
|
||||
end
|
||||
|
||||
def enabled
|
||||
true
|
||||
end
|
||||
|
||||
def tar
|
||||
if system(*%w[gtar --version], out: '/dev/null')
|
||||
# It looks like we can get GNU tar by running 'gtar'
|
||||
|
|
@ -146,7 +145,7 @@ module Backup
|
|||
end
|
||||
end
|
||||
|
||||
def raise_custom_error
|
||||
def raise_custom_error(backup_tarball)
|
||||
raise FileBackupError.new(app_files_realpath, backup_tarball)
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ module Backup
|
|||
@storage_parallelism = storage_parallelism
|
||||
end
|
||||
|
||||
def start(type)
|
||||
def start(type, backup_repos_path)
|
||||
raise Error, 'already started' if started?
|
||||
|
||||
command = case type
|
||||
|
|
@ -93,10 +93,6 @@ module Backup
|
|||
@thread.present?
|
||||
end
|
||||
|
||||
def backup_repos_path
|
||||
File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
|
||||
end
|
||||
|
||||
def bin_path
|
||||
File.absolute_path(Gitlab.config.backup.gitaly_backup_path)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -7,10 +7,11 @@ module Backup
|
|||
@progress = progress
|
||||
end
|
||||
|
||||
def start(type)
|
||||
def start(type, backup_repos_path)
|
||||
raise Error, 'already started' if @type
|
||||
|
||||
@type = type
|
||||
@backup_repos_path = backup_repos_path
|
||||
case type
|
||||
when :create
|
||||
FileUtils.rm_rf(backup_repos_path)
|
||||
|
|
@ -31,7 +32,7 @@ module Backup
|
|||
backup_restore = BackupRestore.new(
|
||||
progress,
|
||||
repository_type.repository_for(container),
|
||||
backup_repos_path
|
||||
@backup_repos_path
|
||||
)
|
||||
|
||||
case @type
|
||||
|
|
@ -52,10 +53,6 @@ module Backup
|
|||
|
||||
attr_reader :progress
|
||||
|
||||
def backup_repos_path
|
||||
@backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
|
||||
end
|
||||
|
||||
class BackupRestore
|
||||
attr_accessor :progress, :repository, :backup_repos_path
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class Lfs < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('lfs', Settings.lfs.storage_path)
|
||||
super(progress, 'lfs', Settings.lfs.storage_path)
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('lfs objects')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,37 +2,77 @@
|
|||
|
||||
module Backup
|
||||
class Manager
|
||||
ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry packages].freeze
|
||||
FOLDERS_TO_BACKUP = %w[repositories db].freeze
|
||||
FILE_NAME_SUFFIX = '_gitlab_backup.tar'
|
||||
MANIFEST_NAME = 'backup_information.yml'
|
||||
|
||||
TaskDefinition = Struct.new(
|
||||
:destination_path, # Where the task should put its backup file/dir.
|
||||
:destination_optional, # `true` if the destination might not exist on a successful backup.
|
||||
:cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified.
|
||||
:task,
|
||||
keyword_init: true
|
||||
)
|
||||
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
def initialize(progress, definitions: nil)
|
||||
@progress = progress
|
||||
|
||||
max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
|
||||
max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
|
||||
force = ENV['force'] == 'yes'
|
||||
|
||||
@tasks = {
|
||||
'db' => Database.new(progress),
|
||||
'repositories' => Repositories.new(progress,
|
||||
strategy: repository_backup_strategy,
|
||||
max_concurrency: max_concurrency,
|
||||
max_storage_concurrency: max_storage_concurrency),
|
||||
'uploads' => Uploads.new(progress),
|
||||
'builds' => Builds.new(progress),
|
||||
'artifacts' => Artifacts.new(progress),
|
||||
'pages' => Pages.new(progress),
|
||||
'lfs' => Lfs.new(progress),
|
||||
'terraform_state' => TerraformState.new(progress),
|
||||
'registry' => Registry.new(progress),
|
||||
'packages' => Packages.new(progress)
|
||||
@definitions = definitions || {
|
||||
'db' => TaskDefinition.new(
|
||||
destination_path: 'db/database.sql.gz',
|
||||
cleanup_path: 'db',
|
||||
task: Database.new(progress, force: force)
|
||||
),
|
||||
'repositories' => TaskDefinition.new(
|
||||
destination_path: 'repositories',
|
||||
destination_optional: true,
|
||||
task: Repositories.new(progress,
|
||||
strategy: repository_backup_strategy,
|
||||
max_concurrency: max_concurrency,
|
||||
max_storage_concurrency: max_storage_concurrency)
|
||||
),
|
||||
'uploads' => TaskDefinition.new(
|
||||
destination_path: 'uploads.tar.gz',
|
||||
task: Uploads.new(progress)
|
||||
),
|
||||
'builds' => TaskDefinition.new(
|
||||
destination_path: 'builds.tar.gz',
|
||||
task: Builds.new(progress)
|
||||
),
|
||||
'artifacts' => TaskDefinition.new(
|
||||
destination_path: 'artifacts.tar.gz',
|
||||
task: Artifacts.new(progress)
|
||||
),
|
||||
'pages' => TaskDefinition.new(
|
||||
destination_path: 'pages.tar.gz',
|
||||
task: Pages.new(progress)
|
||||
),
|
||||
'lfs' => TaskDefinition.new(
|
||||
destination_path: 'lfs.tar.gz',
|
||||
task: Lfs.new(progress)
|
||||
),
|
||||
'terraform_state' => TaskDefinition.new(
|
||||
destination_path: 'terraform_state.tar.gz',
|
||||
task: TerraformState.new(progress)
|
||||
),
|
||||
'registry' => TaskDefinition.new(
|
||||
destination_path: 'registry.tar.gz',
|
||||
task: Registry.new(progress)
|
||||
),
|
||||
'packages' => TaskDefinition.new(
|
||||
destination_path: 'packages.tar.gz',
|
||||
task: Packages.new(progress)
|
||||
)
|
||||
}.freeze
|
||||
end
|
||||
|
||||
def create
|
||||
@tasks.keys.each do |task_name|
|
||||
@definitions.keys.each do |task_name|
|
||||
run_create_task(task_name)
|
||||
end
|
||||
|
||||
|
|
@ -54,11 +94,11 @@ module Backup
|
|||
end
|
||||
|
||||
def run_create_task(task_name)
|
||||
task = @tasks[task_name]
|
||||
definition = @definitions[task_name]
|
||||
|
||||
puts_time "Dumping #{task.human_name} ... ".color(:blue)
|
||||
puts_time "Dumping #{definition.task.human_name} ... ".color(:blue)
|
||||
|
||||
unless task.enabled
|
||||
unless definition.task.enabled
|
||||
puts_time "[DISABLED]".color(:cyan)
|
||||
return
|
||||
end
|
||||
|
|
@ -68,7 +108,8 @@ module Backup
|
|||
return
|
||||
end
|
||||
|
||||
task.dump
|
||||
definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path))
|
||||
|
||||
puts_time "done".color(:green)
|
||||
|
||||
rescue Backup::DatabaseBackupError, Backup::FileBackupError => e
|
||||
|
|
@ -79,39 +120,7 @@ module Backup
|
|||
cleanup_required = unpack
|
||||
verify_backup_version
|
||||
|
||||
unless skipped?('db')
|
||||
begin
|
||||
unless ENV['force'] == 'yes'
|
||||
warning = <<-MSG.strip_heredoc
|
||||
Be sure to stop Puma, Sidekiq, and any other process that
|
||||
connects to the database before proceeding. For Omnibus
|
||||
installs, see the following link for more information:
|
||||
https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
|
||||
|
||||
Before restoring the database, we will remove all existing
|
||||
tables to avoid future upgrade problems. Be aware that if you have
|
||||
custom tables in the GitLab database these tables and all data will be
|
||||
removed.
|
||||
MSG
|
||||
puts warning.color(:red)
|
||||
Gitlab::TaskHelpers.ask_to_continue
|
||||
puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
|
||||
sleep(5)
|
||||
end
|
||||
|
||||
# Drop all tables Load the schema to ensure we don't have any newer tables
|
||||
# hanging out from a failed upgrade
|
||||
puts_time 'Cleaning the database ... '.color(:blue)
|
||||
Rake::Task['gitlab:db:drop_tables'].invoke
|
||||
puts_time 'done'.color(:green)
|
||||
run_restore_task('db')
|
||||
rescue Gitlab::TaskAbortedByUserError
|
||||
puts "Quitting...".color(:red)
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
@tasks.except('db').keys.each do |task_name|
|
||||
@definitions.keys.each do |task_name|
|
||||
run_restore_task(task_name) unless skipped?(task_name)
|
||||
end
|
||||
|
||||
|
|
@ -130,25 +139,44 @@ module Backup
|
|||
end
|
||||
|
||||
def run_restore_task(task_name)
|
||||
task = @tasks[task_name]
|
||||
definition = @definitions[task_name]
|
||||
|
||||
puts_time "Restoring #{task.human_name} ... ".color(:blue)
|
||||
puts_time "Restoring #{definition.task.human_name} ... ".color(:blue)
|
||||
|
||||
unless task.enabled
|
||||
unless definition.task.enabled
|
||||
puts_time "[DISABLED]".color(:cyan)
|
||||
return
|
||||
end
|
||||
|
||||
task.restore
|
||||
warning = definition.task.pre_restore_warning
|
||||
if warning.present?
|
||||
puts_time warning.color(:red)
|
||||
Gitlab::TaskHelpers.ask_to_continue
|
||||
end
|
||||
|
||||
definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path))
|
||||
|
||||
puts_time "done".color(:green)
|
||||
|
||||
warning = definition.task.post_restore_warning
|
||||
if warning.present?
|
||||
puts_time warning.color(:red)
|
||||
Gitlab::TaskHelpers.ask_to_continue
|
||||
end
|
||||
|
||||
rescue Gitlab::TaskAbortedByUserError
|
||||
puts_time "Quitting...".color(:red)
|
||||
exit 1
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def write_info
|
||||
# Make sure there is a connection
|
||||
ActiveRecord::Base.connection.reconnect!
|
||||
|
||||
Dir.chdir(backup_path) do
|
||||
File.open("#{backup_path}/backup_information.yml", "w+") do |file|
|
||||
File.open("#{backup_path}/#{MANIFEST_NAME}", "w+") do |file|
|
||||
file << backup_information.to_yaml.gsub(/^---\n/, '')
|
||||
end
|
||||
end
|
||||
|
|
@ -182,8 +210,11 @@ module Backup
|
|||
upload = directory.files.create(create_attributes)
|
||||
|
||||
if upload
|
||||
progress.puts "done".color(:green)
|
||||
upload
|
||||
if upload.respond_to?(:encryption) && upload.encryption
|
||||
progress.puts "done (encrypted with #{upload.encryption})".color(:green)
|
||||
else
|
||||
progress.puts "done".color(:green)
|
||||
end
|
||||
else
|
||||
puts "uploading backup to #{remote_directory} failed".color(:red)
|
||||
raise Backup::Error, 'Backup failed'
|
||||
|
|
@ -193,18 +224,19 @@ module Backup
|
|||
def cleanup
|
||||
progress.print "Deleting tmp directories ... "
|
||||
|
||||
backup_contents.each do |dir|
|
||||
next unless File.exist?(File.join(backup_path, dir))
|
||||
|
||||
if FileUtils.rm_rf(File.join(backup_path, dir))
|
||||
progress.puts "done".color(:green)
|
||||
else
|
||||
puts "deleting tmp directory '#{dir}' failed".color(:red)
|
||||
raise Backup::Error, 'Backup failed'
|
||||
end
|
||||
remove_backup_path(MANIFEST_NAME)
|
||||
@definitions.each do |_, definition|
|
||||
remove_backup_path(definition.cleanup_path || definition.destination_path)
|
||||
end
|
||||
end
|
||||
|
||||
def remove_backup_path(path)
|
||||
return unless File.exist?(File.join(backup_path, path))
|
||||
|
||||
FileUtils.rm_rf(File.join(backup_path, path))
|
||||
progress.puts "done".color(:green)
|
||||
end
|
||||
|
||||
def remove_tmp
|
||||
# delete tmp inside backups
|
||||
progress.print "Deleting backups/tmp ... "
|
||||
|
|
@ -322,10 +354,8 @@ module Backup
|
|||
settings[:skipped] && settings[:skipped].include?(item) || !enabled_task?(item)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def enabled_task?(task_name)
|
||||
@tasks[task_name].enabled
|
||||
@definitions[task_name].task.enabled
|
||||
end
|
||||
|
||||
def backup_file?(file)
|
||||
|
|
@ -333,7 +363,7 @@ module Backup
|
|||
end
|
||||
|
||||
def non_tarred_backup?
|
||||
File.exist?(File.join(backup_path, 'backup_information.yml'))
|
||||
File.exist?(File.join(backup_path, MANIFEST_NAME))
|
||||
end
|
||||
|
||||
def backup_path
|
||||
|
|
@ -380,19 +410,14 @@ module Backup
|
|||
end
|
||||
|
||||
def backup_contents
|
||||
folders_to_backup + archives_to_backup + ["backup_information.yml"]
|
||||
end
|
||||
|
||||
def archives_to_backup
|
||||
ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact
|
||||
end
|
||||
|
||||
def folders_to_backup
|
||||
FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) }
|
||||
[MANIFEST_NAME] + @definitions.reject do |name, definition|
|
||||
skipped?(name) ||
|
||||
(definition.destination_optional && !File.exist?(File.join(backup_path, definition.destination_path)))
|
||||
end.values.map(&:destination_path)
|
||||
end
|
||||
|
||||
def settings
|
||||
@settings ||= YAML.load_file("backup_information.yml")
|
||||
@settings ||= YAML.load_file(MANIFEST_NAME)
|
||||
end
|
||||
|
||||
def tar_file
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class Packages < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('packages', Settings.packages.storage_path, excludes: ['tmp'])
|
||||
super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp'])
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('packages')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,14 +6,11 @@ module Backup
|
|||
# if some of these files are still there, we don't need them in the backup
|
||||
LEGACY_PAGES_TMP_PATH = '@pages.tmp'
|
||||
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
|
||||
super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('pages')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,18 +2,16 @@
|
|||
|
||||
module Backup
|
||||
class Registry < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('registry', Settings.registry.path)
|
||||
super(progress, 'registry', Settings.registry.path)
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('container registry images')
|
||||
end
|
||||
|
||||
override :enabled
|
||||
def enabled
|
||||
Gitlab.config.registry.enabled
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,16 +3,20 @@
|
|||
require 'yaml'
|
||||
|
||||
module Backup
|
||||
class Repositories
|
||||
class Repositories < Task
|
||||
extend ::Gitlab::Utils::Override
|
||||
|
||||
def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1)
|
||||
@progress = progress
|
||||
super(progress)
|
||||
|
||||
@strategy = strategy
|
||||
@max_concurrency = max_concurrency
|
||||
@max_storage_concurrency = max_storage_concurrency
|
||||
end
|
||||
|
||||
def dump
|
||||
strategy.start(:create)
|
||||
override :dump
|
||||
def dump(path)
|
||||
strategy.start(:create, path)
|
||||
|
||||
# gitaly-backup is designed to handle concurrency on its own. So we want
|
||||
# to avoid entering the buggy concurrency code here when gitaly-backup
|
||||
|
|
@ -50,8 +54,9 @@ module Backup
|
|||
strategy.finish!
|
||||
end
|
||||
|
||||
def restore
|
||||
strategy.start(:restore)
|
||||
override :restore
|
||||
def restore(path)
|
||||
strategy.start(:restore, path)
|
||||
enqueue_consecutive
|
||||
|
||||
ensure
|
||||
|
|
@ -61,17 +66,14 @@ module Backup
|
|||
restore_object_pools
|
||||
end
|
||||
|
||||
def enabled
|
||||
true
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('repositories')
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :progress, :strategy, :max_concurrency, :max_storage_concurrency
|
||||
attr_reader :strategy, :max_concurrency, :max_storage_concurrency
|
||||
|
||||
def check_valid_storages!
|
||||
repository_storage_klasses.each do |klass|
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Backup
|
||||
class Task
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
end
|
||||
|
||||
# human readable task name used for logging
|
||||
def human_name
|
||||
raise NotImplementedError
|
||||
end
|
||||
|
||||
# dump task backup to `path`
|
||||
def dump(path)
|
||||
raise NotImplementedError
|
||||
end
|
||||
|
||||
# restore task backup from `path`
|
||||
def restore(path)
|
||||
raise NotImplementedError
|
||||
end
|
||||
|
||||
# a string returned here will be displayed to the user before calling #restore
|
||||
def pre_restore_warning
|
||||
end
|
||||
|
||||
# a string returned here will be displayed to the user after calling #restore
|
||||
def post_restore_warning
|
||||
end
|
||||
|
||||
# returns `true` when the task should be used
|
||||
def enabled
|
||||
true
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :progress
|
||||
|
||||
def puts_time(msg)
|
||||
progress.puts "#{Time.zone.now} -- #{msg}"
|
||||
Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class TerraformState < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
|
||||
super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('terraform states')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
module Backup
|
||||
class Uploads < Backup::Files
|
||||
attr_reader :progress
|
||||
|
||||
def initialize(progress)
|
||||
@progress = progress
|
||||
|
||||
super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
|
||||
super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
|
||||
end
|
||||
|
||||
override :human_name
|
||||
def human_name
|
||||
_('uploads')
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
#!/usr/bin/env ruby
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rake'
|
||||
|
||||
host = ARGV.shift
|
||||
ENV['CONTRACT_HOST'] ||= host
|
||||
|
||||
list = []
|
||||
|
||||
loop do
|
||||
keyword = ARGV.shift
|
||||
case keyword
|
||||
when '--mr'
|
||||
ENV['CONTRACT_MR'] ||= ARGV.shift
|
||||
list.push 'test:merge_request'
|
||||
else
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
app = Rake.application
|
||||
|
||||
Dir.chdir('contracts/provider') do
|
||||
app.init
|
||||
app.add_import 'Rakefile'
|
||||
app.load_rakefile
|
||||
|
||||
list.each do |element|
|
||||
app[element].invoke
|
||||
end
|
||||
end
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'pact/tasks/verification_task'
|
||||
|
||||
Pact::VerificationTask.new(:metadata) do |pact|
|
||||
pact.uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json', pact_helper: './spec/metadata_helper.rb'
|
||||
end
|
||||
|
||||
Pact::VerificationTask.new(:discussions) do |pact|
|
||||
pact.uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json', pact_helper: './spec/discussions_helper.rb'
|
||||
end
|
||||
|
||||
Pact::VerificationTask.new(:diffs) do |pact|
|
||||
pact.uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json', pact_helper: './spec/diffs_helper.rb'
|
||||
end
|
||||
|
||||
task 'test:merge_request' => ['pact:verify:metadata', 'pact:verify:discussions', 'pact:verify:diffs']
|
||||
|
|
@ -1,25 +1,23 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'faraday'
|
||||
module Provider
|
||||
module Environments
|
||||
class Base
|
||||
attr_writer :base_url, :merge_request
|
||||
|
||||
module Environments
|
||||
class Base
|
||||
attr_writer :base_url, :merge_request
|
||||
def call(env)
|
||||
@payload
|
||||
end
|
||||
|
||||
def call(env)
|
||||
@payload
|
||||
end
|
||||
def http(endpoint)
|
||||
Faraday.default_adapter = :net_http
|
||||
response = Faraday.get(@base_url + endpoint)
|
||||
@payload = [response.status, response.headers, [response.body]]
|
||||
self
|
||||
end
|
||||
|
||||
def http(endpoint)
|
||||
Faraday.default_adapter = :net_http
|
||||
response = Faraday.get(@base_url + endpoint)
|
||||
@payload = [response.status, response.headers, [response.body]]
|
||||
self
|
||||
end
|
||||
|
||||
def merge_request(endpoint)
|
||||
if endpoint.include? '.json'
|
||||
http(@merge_request + endpoint)
|
||||
def merge_request(endpoint)
|
||||
http(@merge_request + endpoint) if endpoint.include? '.json'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative './base'
|
||||
|
||||
module Environments
|
||||
class Local < Base
|
||||
def initialize
|
||||
@base_url = ENV['CONTRACT_HOST']
|
||||
@merge_request = ENV['CONTRACT_MR']
|
||||
module Provider
|
||||
module Environments
|
||||
class Local < Base
|
||||
def initialize
|
||||
@base_url = ENV['CONTRACT_HOST']
|
||||
@merge_request = ENV['CONTRACT_MR']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../environments/local'
|
||||
require_relative '../spec_helper'
|
||||
|
||||
module DiffsHelper
|
||||
local = Environments::Local.new
|
||||
module Provider
|
||||
module DiffsHelper
|
||||
local = Environments::Local.new
|
||||
|
||||
Pact.service_provider "Merge Request Diffs Endpoint" do
|
||||
app { local.merge_request('/diffs_batch.json?page=0') }
|
||||
Pact.service_provider "Merge Request Diffs Endpoint" do
|
||||
app { local.merge_request('/diffs_batch.json?page=0') }
|
||||
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json'
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_diffs_endpoint.json'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../environments/local'
|
||||
require_relative '../spec_helper'
|
||||
|
||||
module DiscussionsHelper
|
||||
local = Environments::Local.new
|
||||
module Provider
|
||||
module DiscussionsHelper
|
||||
local = Environments::Local.new
|
||||
|
||||
Pact.service_provider "Merge Request Discussions Endpoint" do
|
||||
app { local.merge_request('/discussions.json') }
|
||||
Pact.service_provider "Merge Request Discussions Endpoint" do
|
||||
app { local.merge_request('/discussions.json') }
|
||||
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json'
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../environments/local'
|
||||
require_relative '../spec_helper'
|
||||
|
||||
module MetadataHelper
|
||||
local = Environments::Local.new
|
||||
module Provider
|
||||
module MetadataHelper
|
||||
local = Environments::Local.new
|
||||
|
||||
Pact.service_provider "Merge Request Metadata Endpoint" do
|
||||
app { local.merge_request('/diffs_metadata.json') }
|
||||
Pact.service_provider "Merge Request Metadata Endpoint" do
|
||||
app { local.merge_request('/diffs_metadata.json') }
|
||||
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json'
|
||||
honours_pact_with 'Merge Request Page' do
|
||||
pact_uri '../contracts/merge_request_page-merge_request_metadata_endpoint.json'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module SpecHelper
|
||||
unless ENV['CONTRACT_HOST']
|
||||
raise(ArgumentError, 'Contract tests require CONTRACT_HOST environment variable to be set!')
|
||||
end
|
||||
|
||||
require_relative '../../../config/bundler_setup'
|
||||
Bundler.require(:default)
|
||||
|
||||
root = File.expand_path('../', __dir__)
|
||||
|
||||
loader = Zeitwerk::Loader.new
|
||||
loader.push_dir(root)
|
||||
|
||||
loader.ignore("#{root}/consumer")
|
||||
loader.ignore("#{root}/contracts")
|
||||
|
||||
loader.collapse("#{root}/provider/spec")
|
||||
|
||||
loader.setup
|
||||
end
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'pact/tasks/verification_task'
|
||||
|
||||
contracts = File.expand_path('../contracts', __dir__)
|
||||
provider = File.expand_path('provider', contracts)
|
||||
|
||||
# rubocop:disable Rails/RakeEnvironment
|
||||
namespace :contracts do
|
||||
namespace :mr do
|
||||
Pact::VerificationTask.new(:metadata) do |pact|
|
||||
pact.uri(
|
||||
"#{contracts}/contracts/merge_request_page-merge_request_metadata_endpoint.json",
|
||||
pact_helper: "#{provider}/spec/metadata_helper.rb"
|
||||
)
|
||||
end
|
||||
|
||||
Pact::VerificationTask.new(:discussions) do |pact|
|
||||
pact.uri(
|
||||
"#{contracts}/contracts/merge_request_page-merge_request_discussions_endpoint.json",
|
||||
pact_helper: "#{provider}/spec/discussions_helper.rb"
|
||||
)
|
||||
end
|
||||
|
||||
Pact::VerificationTask.new(:diffs) do |pact|
|
||||
pact.uri(
|
||||
"#{contracts}/contracts/merge_request_page-merge_request_diffs_endpoint.json",
|
||||
pact_helper: "#{provider}/spec/diffs_helper.rb"
|
||||
)
|
||||
end
|
||||
|
||||
desc 'Run all merge request contract tests'
|
||||
task 'test:merge_request', :contract_mr do |_t, arg|
|
||||
raise(ArgumentError, 'Merge request contract tests require contract_mr to be set') unless arg[:contract_mr]
|
||||
|
||||
ENV['CONTRACT_MR'] = arg[:contract_mr]
|
||||
errors = %w[metadata discussions diffs].each_with_object([]) do |task, err|
|
||||
Rake::Task["contracts:mr:pact:verify:#{task}"].execute
|
||||
rescue StandardError, SystemExit
|
||||
err << "contracts:mr:pact:verify:#{task}"
|
||||
end
|
||||
|
||||
raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty?
|
||||
end
|
||||
end
|
||||
end
|
||||
# rubocop:enable Rails/RakeEnvironment
|
||||
|
|
@ -41,7 +41,8 @@ export const getSecurityTrainingProvidersData = (providerOverrides = {}) => {
|
|||
const response = {
|
||||
data: {
|
||||
project: {
|
||||
id: 1,
|
||||
id: 'gid://gitlab/Project/1',
|
||||
__typename: 'Project',
|
||||
securityTrainingProviders,
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ RSpec.describe Backup::Artifacts do
|
|||
expect(backup).to receive(:tar).and_return('blabla-tar')
|
||||
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
|
||||
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
|
||||
backup.dump
|
||||
backup.dump('artifacts.tar.gz')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,25 +6,49 @@ RSpec.describe Backup::Database do
|
|||
let(:progress) { StringIO.new }
|
||||
let(:output) { progress.string }
|
||||
|
||||
before do
|
||||
allow(Gitlab::TaskHelpers).to receive(:ask_to_continue)
|
||||
before(:all) do
|
||||
Rake.application.rake_require 'active_record/railties/databases'
|
||||
Rake.application.rake_require 'tasks/gitlab/backup'
|
||||
Rake.application.rake_require 'tasks/gitlab/shell'
|
||||
Rake.application.rake_require 'tasks/gitlab/db'
|
||||
Rake.application.rake_require 'tasks/cache'
|
||||
end
|
||||
|
||||
describe '#restore' do
|
||||
let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
|
||||
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
|
||||
let(:force) { true }
|
||||
|
||||
subject { described_class.new(progress, filename: data) }
|
||||
subject { described_class.new(progress, force: force) }
|
||||
|
||||
before do
|
||||
allow(subject).to receive(:pg_restore_cmd).and_return(cmd)
|
||||
end
|
||||
|
||||
context 'when not forced' do
|
||||
let(:force) { false }
|
||||
|
||||
it 'warns the user and waits' do
|
||||
expect(subject).to receive(:sleep)
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
subject.restore(data)
|
||||
|
||||
expect(output).to include('Removing all tables. Press `Ctrl-C` within 5 seconds to abort')
|
||||
end
|
||||
|
||||
it 'has a pre restore warning' do
|
||||
expect(subject.pre_restore_warning).not_to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context 'with an empty .gz file' do
|
||||
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
|
||||
|
||||
it 'returns successfully' do
|
||||
subject.restore
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
subject.restore(data)
|
||||
|
||||
expect(output).to include("Restoring PostgreSQL database")
|
||||
expect(output).to include("[DONE]")
|
||||
|
|
@ -36,7 +60,9 @@ RSpec.describe Backup::Database do
|
|||
let(:data) { Rails.root.join("spec/fixtures/big-image.png").to_s }
|
||||
|
||||
it 'raises a backup error' do
|
||||
expect { subject.restore }.to raise_error(Backup::Error)
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
expect { subject.restore(data) }.to raise_error(Backup::Error)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -45,12 +71,15 @@ RSpec.describe Backup::Database do
|
|||
let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\nWARNING: no privileges could be revoked for public\n" }
|
||||
let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
|
||||
|
||||
it 'filters out noise from errors' do
|
||||
subject.restore
|
||||
it 'filters out noise from errors and has a post restore warning' do
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
subject.restore(data)
|
||||
|
||||
expect(output).to include("ERRORS")
|
||||
expect(output).not_to include(noise)
|
||||
expect(output).to include(visible_error)
|
||||
expect(subject.post_restore_warning).not_to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -66,7 +95,9 @@ RSpec.describe Backup::Database do
|
|||
end
|
||||
|
||||
it 'overrides default config values' do
|
||||
subject.restore
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
subject.restore(data)
|
||||
|
||||
expect(output).to include(%("PGHOST"=>"test.example.com"))
|
||||
expect(output).to include(%("PGPASSWORD"=>"donotchange"))
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#restore' do
|
||||
subject { described_class.new('registry', '/var/gitlab-registry') }
|
||||
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
|
||||
|
||||
let(:timestamp) { Time.utc(2017, 3, 22) }
|
||||
|
||||
|
|
@ -58,11 +58,11 @@ RSpec.describe Backup::Files do
|
|||
it 'moves all necessary files' do
|
||||
allow(subject).to receive(:backup_existing_files).and_call_original
|
||||
expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}"))
|
||||
subject.restore
|
||||
subject.restore('registry.tar.gz')
|
||||
end
|
||||
|
||||
it 'raises no errors' do
|
||||
expect { subject.restore }.not_to raise_error
|
||||
expect { subject.restore('registry.tar.gz') }.not_to raise_error
|
||||
end
|
||||
|
||||
it 'calls tar command with unlink' do
|
||||
|
|
@ -70,13 +70,13 @@ RSpec.describe Backup::Files do
|
|||
|
||||
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
|
||||
expect(subject).to receive(:pipeline_succeeded?).and_return(true)
|
||||
subject.restore
|
||||
subject.restore('registry.tar.gz')
|
||||
end
|
||||
|
||||
it 'raises an error on failure' do
|
||||
expect(subject).to receive(:pipeline_succeeded?).and_return(false)
|
||||
|
||||
expect { subject.restore }.to raise_error(/Restore operation failed:/)
|
||||
expect { subject.restore('registry.tar.gz') }.to raise_error(/Restore operation failed:/)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ RSpec.describe Backup::Files do
|
|||
|
||||
it 'shows error message' do
|
||||
expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry")
|
||||
subject.restore
|
||||
subject.restore('registry.tar.gz')
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -104,13 +104,13 @@ RSpec.describe Backup::Files do
|
|||
expect(subject).to receive(:resource_busy_error).with("/var/gitlab-registry")
|
||||
.and_call_original
|
||||
|
||||
expect { subject.restore }.to raise_error(/is a mountpoint/)
|
||||
expect { subject.restore('registry.tar.gz') }.to raise_error(/is a mountpoint/)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#dump' do
|
||||
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
|
||||
subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
|
||||
|
||||
before do
|
||||
allow(subject).to receive(:run_pipeline!).and_return([[true, true], ''])
|
||||
|
|
@ -118,14 +118,14 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
it 'raises no errors' do
|
||||
expect { subject.dump }.not_to raise_error
|
||||
expect { subject.dump('registry.tar.gz') }.not_to raise_error
|
||||
end
|
||||
|
||||
it 'excludes tmp dirs from archive' do
|
||||
expect(subject).to receive(:tar).and_return('blabla-tar')
|
||||
|
||||
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args)
|
||||
subject.dump
|
||||
subject.dump('registry.tar.gz')
|
||||
end
|
||||
|
||||
it 'raises an error on failure' do
|
||||
|
|
@ -133,7 +133,7 @@ RSpec.describe Backup::Files do
|
|||
expect(subject).to receive(:pipeline_succeeded?).and_return(false)
|
||||
|
||||
expect do
|
||||
subject.dump
|
||||
subject.dump('registry.tar.gz')
|
||||
end.to raise_error(/Failed to create compressed file/)
|
||||
end
|
||||
|
||||
|
|
@ -149,7 +149,7 @@ RSpec.describe Backup::Files do
|
|||
.with(%w(rsync -a --delete --exclude=lost+found --exclude=/gitlab-pages/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
|
||||
.and_return(['', 0])
|
||||
|
||||
subject.dump
|
||||
subject.dump('registry.tar.gz')
|
||||
end
|
||||
|
||||
it 'retries if rsync fails due to vanishing files' do
|
||||
|
|
@ -158,7 +158,7 @@ RSpec.describe Backup::Files do
|
|||
.and_return(['rsync failed', 24], ['', 0])
|
||||
|
||||
expect do
|
||||
subject.dump
|
||||
subject.dump('registry.tar.gz')
|
||||
end.to output(/files vanished during rsync, retrying/).to_stdout
|
||||
end
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ RSpec.describe Backup::Files do
|
|||
.and_return(['rsync failed', 1])
|
||||
|
||||
expect do
|
||||
subject.dump
|
||||
subject.dump('registry.tar.gz')
|
||||
end.to output(/rsync failed/).to_stdout
|
||||
.and raise_error(/Failed to create compressed file/)
|
||||
end
|
||||
|
|
@ -176,7 +176,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#exclude_dirs' do
|
||||
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
|
||||
subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
|
||||
|
||||
it 'prepends a leading dot slash to tar excludes' do
|
||||
expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp'])
|
||||
|
|
@ -188,7 +188,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#run_pipeline!' do
|
||||
subject { described_class.new('registry', '/var/gitlab-registry') }
|
||||
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
|
||||
|
||||
it 'executes an Open3.pipeline for cmd_list' do
|
||||
expect(Open3).to receive(:pipeline).with(%w[whew command], %w[another cmd], any_args)
|
||||
|
|
@ -222,7 +222,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#pipeline_succeeded?' do
|
||||
subject { described_class.new('registry', '/var/gitlab-registry') }
|
||||
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
|
||||
|
||||
it 'returns true if both tar and gzip succeeeded' do
|
||||
expect(
|
||||
|
|
@ -262,7 +262,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#tar_ignore_non_success?' do
|
||||
subject { described_class.new('registry', '/var/gitlab-registry') }
|
||||
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
|
||||
|
||||
context 'if `tar` command exits with 1 exitstatus' do
|
||||
it 'returns true' do
|
||||
|
|
@ -310,7 +310,7 @@ RSpec.describe Backup::Files do
|
|||
end
|
||||
|
||||
describe '#noncritical_warning?' do
|
||||
subject { described_class.new('registry', '/var/gitlab-registry') }
|
||||
subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
|
||||
|
||||
it 'returns true if given text matches noncritical warnings list' do
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ require 'spec_helper'
|
|||
RSpec.describe Backup::GitalyBackup do
|
||||
let(:max_parallelism) { nil }
|
||||
let(:storage_parallelism) { nil }
|
||||
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
|
||||
|
||||
let(:progress) do
|
||||
Tempfile.new('progress').tap do |progress|
|
||||
|
|
@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
|
||||
context 'unknown' do
|
||||
it 'fails to start unknown' do
|
||||
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
|
||||
expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -42,7 +43,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.enqueue(project, Gitlab::GlRepository::WIKI)
|
||||
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -50,11 +51,11 @@ RSpec.describe Backup::GitalyBackup do
|
|||
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
|
||||
subject.finish!
|
||||
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
|
||||
expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
|
||||
end
|
||||
|
||||
context 'parallel option set' do
|
||||
|
|
@ -63,7 +64,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'passes parallel option through' do
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.finish!
|
||||
end
|
||||
end
|
||||
|
|
@ -74,7 +75,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'passes parallel option through' do
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.finish!
|
||||
end
|
||||
end
|
||||
|
|
@ -82,7 +83,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'raises when the exit code not zero' do
|
||||
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
|
||||
end
|
||||
end
|
||||
|
|
@ -114,7 +115,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'passes through SSL envs' do
|
||||
expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.finish!
|
||||
end
|
||||
end
|
||||
|
|
@ -139,7 +140,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.enqueue(project, Gitlab::GlRepository::WIKI)
|
||||
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -162,7 +163,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'passes parallel option through' do
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.finish!
|
||||
end
|
||||
end
|
||||
|
|
@ -173,7 +174,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'passes parallel option through' do
|
||||
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.finish!
|
||||
end
|
||||
end
|
||||
|
|
@ -181,7 +182,7 @@ RSpec.describe Backup::GitalyBackup do
|
|||
it 'raises when the exit code not zero' do
|
||||
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ require 'spec_helper'
|
|||
|
||||
RSpec.describe Backup::GitalyRpcBackup do
|
||||
let(:progress) { spy(:stdout) }
|
||||
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
|
||||
|
||||
subject { described_class.new(progress) }
|
||||
|
||||
|
|
@ -14,7 +15,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
|
||||
context 'unknown' do
|
||||
it 'fails to start unknown' do
|
||||
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
|
||||
expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
project_snippet = create(:project_snippet, :repository, project: project)
|
||||
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
|
||||
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.enqueue(project, Gitlab::GlRepository::WIKI)
|
||||
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -35,11 +36,11 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
|
||||
subject.finish!
|
||||
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
|
||||
expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
|
||||
expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
|
||||
expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
|
||||
end
|
||||
|
||||
context 'failure' do
|
||||
|
|
@ -50,7 +51,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
end
|
||||
|
||||
it 'logs an appropriate message', :aggregate_failures do
|
||||
subject.start(:create)
|
||||
subject.start(:create, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.finish!
|
||||
|
||||
|
|
@ -90,7 +91,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
|
||||
copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.enqueue(project, Gitlab::GlRepository::WIKI)
|
||||
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -123,7 +124,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
repository
|
||||
end
|
||||
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.enqueue(project, Gitlab::GlRepository::WIKI)
|
||||
subject.enqueue(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -141,7 +142,7 @@ RSpec.describe Backup::GitalyRpcBackup do
|
|||
end
|
||||
|
||||
it 'logs an appropriate message', :aggregate_failures do
|
||||
subject.start(:restore)
|
||||
subject.start(:restore, destination)
|
||||
subject.enqueue(project, Gitlab::GlRepository::PROJECT)
|
||||
subject.finish!
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ RSpec.describe Backup::Lfs do
|
|||
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
|
||||
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
|
||||
|
||||
backup.dump
|
||||
backup.dump('lfs.tar.gz')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -21,7 +21,7 @@ RSpec.shared_examples 'backup object' do |setting|
|
|||
expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
|
||||
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
|
||||
|
||||
backup.dump
|
||||
backup.dump('backup_object.tar.gz')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ RSpec.describe Backup::Pages do
|
|||
expect(subject).to receive(:tar).and_return('blabla-tar')
|
||||
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
|
||||
expect(subject).to receive(:pipeline_succeeded?).and_return(true)
|
||||
subject.dump
|
||||
subject.dump('pages.tar.gz')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ RSpec.describe Backup::Repositories do
|
|||
let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
|
||||
let(:max_concurrency) { 1 }
|
||||
let(:max_storage_concurrency) { 1 }
|
||||
let(:destination) { 'repositories' }
|
||||
|
||||
subject do
|
||||
described_class.new(
|
||||
|
|
@ -26,9 +27,9 @@ RSpec.describe Backup::Repositories do
|
|||
project_snippet = create(:project_snippet, :repository, project: project)
|
||||
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
|
||||
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
|
||||
expect(strategy).to have_received(:start).with(:create)
|
||||
expect(strategy).to have_received(:start).with(:create, destination)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -54,38 +55,38 @@ RSpec.describe Backup::Repositories do
|
|||
it 'creates the expected number of threads' do
|
||||
expect(Thread).not_to receive(:new)
|
||||
|
||||
expect(strategy).to receive(:start).with(:create)
|
||||
expect(strategy).to receive(:start).with(:create, destination)
|
||||
projects.each do |project|
|
||||
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
end
|
||||
expect(strategy).to receive(:finish!)
|
||||
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end
|
||||
|
||||
describe 'command failure' do
|
||||
it 'enqueue_project raises an error' do
|
||||
allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
|
||||
|
||||
expect { subject.dump }.to raise_error(IOError)
|
||||
expect { subject.dump(destination) }.to raise_error(IOError)
|
||||
end
|
||||
|
||||
it 'project query raises an error' do
|
||||
allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
|
||||
|
||||
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
|
||||
expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
|
||||
end
|
||||
end
|
||||
|
||||
it 'avoids N+1 database queries' do
|
||||
control_count = ActiveRecord::QueryRecorder.new do
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end.count
|
||||
|
||||
create_list(:project, 2, :repository)
|
||||
|
||||
expect do
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end.not_to exceed_query_limit(control_count)
|
||||
end
|
||||
end
|
||||
|
|
@ -98,13 +99,13 @@ RSpec.describe Backup::Repositories do
|
|||
it 'enqueues all projects sequentially' do
|
||||
expect(Thread).not_to receive(:new)
|
||||
|
||||
expect(strategy).to receive(:start).with(:create)
|
||||
expect(strategy).to receive(:start).with(:create, destination)
|
||||
projects.each do |project|
|
||||
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
end
|
||||
expect(strategy).to receive(:finish!)
|
||||
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -122,13 +123,13 @@ RSpec.describe Backup::Repositories do
|
|||
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times
|
||||
.and_call_original
|
||||
|
||||
expect(strategy).to receive(:start).with(:create)
|
||||
expect(strategy).to receive(:start).with(:create, destination)
|
||||
projects.each do |project|
|
||||
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
end
|
||||
expect(strategy).to receive(:finish!)
|
||||
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end
|
||||
|
||||
context 'with extra max concurrency' do
|
||||
|
|
@ -139,13 +140,13 @@ RSpec.describe Backup::Repositories do
|
|||
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times
|
||||
.and_call_original
|
||||
|
||||
expect(strategy).to receive(:start).with(:create)
|
||||
expect(strategy).to receive(:start).with(:create, destination)
|
||||
projects.each do |project|
|
||||
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
end
|
||||
expect(strategy).to receive(:finish!)
|
||||
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -153,33 +154,33 @@ RSpec.describe Backup::Repositories do
|
|||
it 'enqueue_project raises an error' do
|
||||
allow(strategy).to receive(:enqueue).and_raise(IOError)
|
||||
|
||||
expect { subject.dump }.to raise_error(IOError)
|
||||
expect { subject.dump(destination) }.to raise_error(IOError)
|
||||
end
|
||||
|
||||
it 'project query raises an error' do
|
||||
allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
|
||||
|
||||
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
|
||||
expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
|
||||
end
|
||||
|
||||
context 'misconfigured storages' do
|
||||
let(:storage_keys) { %w[test_second_storage] }
|
||||
|
||||
it 'raises an error' do
|
||||
expect { subject.dump }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
|
||||
expect { subject.dump(destination) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
it 'avoids N+1 database queries' do
|
||||
control_count = ActiveRecord::QueryRecorder.new do
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end.count
|
||||
|
||||
create_list(:project, 2, :repository)
|
||||
|
||||
expect do
|
||||
subject.dump
|
||||
subject.dump(destination)
|
||||
end.not_to exceed_query_limit(control_count)
|
||||
end
|
||||
end
|
||||
|
|
@ -192,9 +193,9 @@ RSpec.describe Backup::Repositories do
|
|||
let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.first_owner) }
|
||||
|
||||
it 'calls enqueue for each repository type', :aggregate_failures do
|
||||
subject.restore
|
||||
subject.restore(destination)
|
||||
|
||||
expect(strategy).to have_received(:start).with(:restore)
|
||||
expect(strategy).to have_received(:start).with(:restore, destination)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
|
||||
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
|
||||
|
|
@ -208,7 +209,7 @@ RSpec.describe Backup::Repositories do
|
|||
pool_repository = create(:pool_repository, :failed)
|
||||
pool_repository.delete_object_pool
|
||||
|
||||
subject.restore
|
||||
subject.restore(destination)
|
||||
|
||||
pool_repository.reload
|
||||
expect(pool_repository).not_to be_failed
|
||||
|
|
@ -219,7 +220,7 @@ RSpec.describe Backup::Repositories do
|
|||
pool_repository = create(:pool_repository, state: :obsolete)
|
||||
pool_repository.update_column(:source_project_id, nil)
|
||||
|
||||
subject.restore
|
||||
subject.restore(destination)
|
||||
|
||||
pool_repository.reload
|
||||
expect(pool_repository).to be_obsolete
|
||||
|
|
@ -236,14 +237,14 @@ RSpec.describe Backup::Repositories do
|
|||
end
|
||||
|
||||
it 'shows the appropriate error' do
|
||||
subject.restore
|
||||
subject.restore(destination)
|
||||
|
||||
expect(progress).to have_received(:puts).with("Snippet #{personal_snippet.full_path} can't be restored: Repository has more than one branch")
|
||||
expect(progress).to have_received(:puts).with("Snippet #{project_snippet.full_path} can't be restored: Repository has more than one branch")
|
||||
end
|
||||
|
||||
it 'removes the snippets from the DB' do
|
||||
expect { subject.restore }.to change(PersonalSnippet, :count).by(-1)
|
||||
expect { subject.restore(destination) }.to change(PersonalSnippet, :count).by(-1)
|
||||
.and change(ProjectSnippet, :count).by(-1)
|
||||
.and change(SnippetRepository, :count).by(-2)
|
||||
end
|
||||
|
|
@ -253,7 +254,7 @@ RSpec.describe Backup::Repositories do
|
|||
shard_name = personal_snippet.repository.shard
|
||||
path = personal_snippet.disk_path + '.git'
|
||||
|
||||
subject.restore
|
||||
subject.restore(destination)
|
||||
|
||||
expect(gitlab_shell.repository_exists?(shard_name, path)).to eq false
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'spec_helper'
|
||||
|
||||
RSpec.describe Backup::Task do
|
||||
let(:progress) { StringIO.new }
|
||||
|
||||
subject { described_class.new(progress) }
|
||||
|
||||
describe '#human_name' do
|
||||
it 'must be implemented by the subclass' do
|
||||
expect { subject.human_name }.to raise_error(NotImplementedError)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#dump' do
|
||||
it 'must be implemented by the subclass' do
|
||||
expect { subject.dump('some/path') }.to raise_error(NotImplementedError)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#restore' do
|
||||
it 'must be implemented by the subclass' do
|
||||
expect { subject.restore('some/path') }.to raise_error(NotImplementedError)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -19,7 +19,7 @@ RSpec.describe Backup::Uploads do
|
|||
expect(backup).to receive(:tar).and_return('blabla-tar')
|
||||
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
|
||||
expect(backup).to receive(:pipeline_succeeded?).and_return(true)
|
||||
backup.dump
|
||||
backup.dump('uploads.tar.gz')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -72,7 +72,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
|
|||
before do
|
||||
allow(YAML).to receive(:load_file)
|
||||
.and_return({ gitlab_version: gitlab_version })
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
expect_next_instance_of(::Backup::Manager) do |instance|
|
||||
backup_types.each do |subtask|
|
||||
expect(instance).to receive(:run_restore_task).with(subtask).ordered
|
||||
|
|
@ -85,10 +84,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
|
|||
it 'invokes restoration on match' do
|
||||
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
|
||||
end
|
||||
|
||||
it 'prints timestamps on messages' do
|
||||
expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout_from_any_process
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -131,8 +126,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
|
|||
allow(YAML).to receive(:load_file)
|
||||
.and_return({ gitlab_version: Gitlab::VERSION })
|
||||
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
|
||||
|
||||
expect_next_instance_of(::Backup::Manager) do |instance|
|
||||
backup_types.each do |subtask|
|
||||
expect(instance).to receive(:run_restore_task).with(subtask).ordered
|
||||
|
|
@ -486,7 +479,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
|
|||
allow(Rake::Task['gitlab:shell:setup'])
|
||||
.to receive(:invoke).and_return(true)
|
||||
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
|
||||
expect_next_instance_of(::Backup::Manager) do |instance|
|
||||
(backup_types - %w{repositories uploads}).each do |subtask|
|
||||
expect(instance).to receive(:run_restore_task).with(subtask).ordered
|
||||
|
|
@ -531,7 +523,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
|
|||
allow(Rake::Task['gitlab:shell:setup'])
|
||||
.to receive(:invoke).and_return(true)
|
||||
|
||||
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
|
||||
expect_next_instance_of(::Backup::Manager) do |instance|
|
||||
backup_types.each do |subtask|
|
||||
expect(instance).to receive(:run_restore_task).with(subtask).ordered
|
||||
|
|
|
|||
Loading…
Reference in New Issue