Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-05-09 06:08:14 +00:00
parent e33183d782
commit 477c7a6cc1
14 changed files with 277 additions and 53 deletions

View file

@ -21,6 +21,8 @@ class Admin::BackgroundMigrationsController < Admin::ApplicationController
def show
@migration = batched_migration_class.find(params[:id])
@failed_jobs = @migration.batched_jobs.with_status(:failed).page(params[:page])
end
def pause

View file

@ -0,0 +1,9 @@
%tr{ role: 'row' }
%td{ role: 'cell', data: { label: _('Id') } }
= job.id
%td{ role: 'cell', data: { label: s_('BackgroundMigrations|Started at') } }
= job.started_at
%td{ role: 'cell', data: { label: s_('BackgroundMigrations|Finished at') } }
= job.finished_at
%td{ role: 'cell', data: { label: s_('BackgroundMigrations|Batch size') } }
= job.batch_size

View file

@ -2,6 +2,8 @@
- breadcrumb_title @migration.job_class_name + ': ' + @migration.table_name
- page_title @migration.job_class_name , _('Background Migrations')
%h3= @migration.job_class_name + ': ' + @migration.table_name
.tab-content.gl-tab-content
.tab-pane.active{ role: 'tabpanel' }
%table.table.b-table.gl-table.b-table-stacked-md{ role: 'table' }
@ -17,6 +19,20 @@
%th.table-th-transparent.border-bottom{ role: 'cell' }= _('Created on')
%th.table-th-transparent.border-bottom{ role: 'cell' }= _('Last updated')
%th.table-th-transparent.border-bottom{ role: 'cell' }= _('Status')
%th.table-th-transparent.border-bottom{ role: 'cell' }
%tbody{ role: 'rowgroup' }
= render partial: 'migration_full_information', migration: @migration
- if @migration.batched_jobs.with_status(:failed).any?
%h5= s_('BackgroundMigrations|Failed jobs:')
%table.table.b-table.gl-table.b-table-stacked-md{ role: 'table' }
%thead{ role: 'rowgroup' }
%tr{ role: 'row' }
%th{ role: 'cell' }= _('Id')
%th{ role: 'cell' }= s_('BackgroundMigrations|Started at')
%th{ role: 'cell' }= s_('BackgroundMigrations|Finished at')
%th{ role: 'cell' }= s_('BackgroundMigrations|Batch size')
%tbody{ role: 'rowgroup' }
= render partial: 'job', collection: @failed_jobs
= paginate_collection @failed_jobs

View file

@ -3,7 +3,7 @@ table_name: issue_customer_relations_contacts
classes:
- CustomerRelations::IssueContact
feature_categories:
- purchase
description: TODO
- team_planning
description: Holds Customer Relations (CRM) Contacts
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71007
milestone: '14.4'

View file

@ -47,11 +47,11 @@ GET /projects/:id/merge_requests/:merge_request_iid/status_checks
> - Introduced in GitLab 14.9, `passed` status to pass external status checks. Introduced [with a flag](../administration/feature_flags.md) named `status_checks_add_status_field`. Disabled by default.
> - Introduced in GitLab 14.9, `failed` status to fail external status checks. Introduced [with a flag](../administration/feature_flags.md) named `status_checks_add_status_field`. Disabled by default.
> - `pass` status to pass checks is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/339039) in GitLab 14.9. Replaced with `passed`.
> - Support for `failed` and `passed` [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/353836) in GitLab 15.0.
FLAG:
On self-managed GitLab, by default setting `passed` instead of `pass` is unavailable. Also, setting `failed` is unavailable by default. To support
setting `passed` and `failed` instead of only `pass`, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named
`status_checks_add_status_field`. On GitLab.com, this feature is not available.
To remove support for `failed` checks (and `pass` instead of `passed`), ask an administrator to
[disable the feature flag](../administration/feature_flags.md) named `status_checks_add_status_field`.
For a single merge request, use the API to inform GitLab that a merge request has passed a check by an external service.
To set the status of an external check, the personal access token used must belong to a user with at least the Developer role on the target project of the merge request.
@ -64,14 +64,13 @@ POST /projects/:id/merge_requests/:merge_request_iid/status_check_responses
**Parameters:**
| Attribute | Type | Required | Description |
| -------------------------- | ------- | -------- | ---------------------------------------------------------------------------- |
| `id` | integer | yes | ID of a project |
| `merge_request_iid` | integer | yes | IID of a merge request |
| `sha` | string | yes | SHA at `HEAD` of the source branch |
| `external_status_check_id` | integer | yes | ID of an external status check |
| `status` | string | no | Set to `passed` to pass the check or `failed` to fail it (GitLab 14.9 and later with feature flag `status_checks_add_status_field` enabled) |
| `status` | string | no | Set to `pass` to pass the check (GitLab 14.0 to GitLab 14.8, and GitLab 14.9 and later with feature flag `status_checks_add_status_field` disabled) |
| Attribute | Type | Required | Description |
| -------------------------- | ------- | -------- |----------------------------------------------------------|
| `id` | integer | yes | ID of a project |
| `merge_request_iid` | integer | yes | IID of a merge request |
| `sha` | string | yes | SHA at `HEAD` of the source branch |
| `external_status_check_id` | integer | yes | ID of an external status check |
| `status` | string | no | Set to `passed` to pass the check or `failed` to fail it |
NOTE:
`sha` must be the SHA at the `HEAD` of the merge request's source branch.

View file

@ -379,18 +379,24 @@ sudo -u git -H bundle exec rake gitlab:backup:create GITLAB_BACKUP_MAX_CONCURREN
> - Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default.
> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355945) in GitLab 14.10.
> - `PREVIOUS_BACKUP` option [introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4184) in GitLab 15.0.
FLAG:
On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`.
On GitLab.com, this feature is not available.
Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup
bundle for each repository. There must be an existing backup to create an incremental backup from and this backup will be overwritten. You can use the `BACKUP=timestamp_of_backup` option to choose which backup will be used.
bundle for each repository. There must be an existing backup to create an incremental backup from:
- In GitLab 14.9 and 14.10, use the `BACKUP=<timestamp_of_backup>` option to choose the backup to use. The chosen previous backup is overwritten.
- In GitLab 15.0 and later, use the `PREVIOUS_BACKUP=<timestamp_of_backup>` option to choose the backup to use. By default, a backup file is created
as documented in the [Backup timestamp](#backup-timestamp) section. You can override the `[TIMESTAMP]` portion of the filename by setting the
[`BACKUP` environment variable](#backup-filename).
To create an incremental backup, run:
```shell
sudo gitlab-backup create INCREMENTAL=yes
sudo gitlab-backup create INCREMENTAL=yes PREVIOUS_BACKUP=<timestamp_of_backup>
```
Incremental backups can also be created from [an untarred backup](#skipping-tar-creation) by using `SKIP=tar`:
@ -1399,6 +1405,7 @@ To prepare the new server:
1. Copy the
[SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079)
from the old server to avoid man-in-the-middle attack warnings.
See [Manually replicate the primary sites SSH host keys](../administration/geo/replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) for example steps.
1. [Install and configure GitLab](https://about.gitlab.com/install) except
[incoming email](../administration/incoming_email.md):
1. Install GitLab.

View file

@ -37,9 +37,10 @@ module Backup
def create
if incremental?
unpack
unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP']))
read_backup_information
verify_backup_version
update_backup_information
end
@definitions.keys.each do |task_name|
@ -79,7 +80,7 @@ module Backup
end
puts_time "Dumping #{definition.human_name} ... ".color(:blue)
definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id)
definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), full_backup_id)
puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "done".color(:green)
rescue Backup::DatabaseBackupError, Backup::FileBackupError => e
@ -87,7 +88,7 @@ module Backup
end
def restore
cleanup_required = unpack
cleanup_required = unpack(ENV['BACKUP'])
read_backup_information
verify_backup_version
@ -248,6 +249,17 @@ module Backup
}
end
def update_backup_information
@backup_information.merge!(
full_backup_id: full_backup_id,
db_version: ActiveRecord::Migrator.current_version.to_s,
backup_created_at: Time.zone.now,
gitlab_version: Gitlab::VERSION,
tar_version: tar_version,
installation_type: Gitlab::INSTALLATION_TYPE
)
end
def backup_information
raise Backup::Error, "#{MANIFEST_NAME} not yet loaded" unless @backup_information
@ -374,8 +386,8 @@ module Backup
end
end
def unpack
if ENV['BACKUP'].blank? && non_tarred_backup?
def unpack(source_backup_id)
if source_backup_id.blank? && non_tarred_backup?
puts_time "Non tarred backup found in #{backup_path}, using that"
return false
@ -387,14 +399,14 @@ module Backup
puts_time "No backups found in #{backup_path}"
puts_time "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
exit 1
elsif backup_file_list.many? && ENV["BACKUP"].nil?
elsif backup_file_list.many? && source_backup_id.nil?
puts_time 'Found more than one backup:'
# print list of available backups
puts_time " " + available_timestamps.join("\n ")
if incremental?
puts_time 'Please specify which one you want to create an incremental backup for:'
puts_time 'rake gitlab:backup:create INCREMENTAL=true BACKUP=timestamp_of_backup'
puts_time 'rake gitlab:backup:create INCREMENTAL=true PREVIOUS_BACKUP=timestamp_of_backup'
else
puts_time 'Please specify which one you want to restore:'
puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
@ -403,8 +415,8 @@ module Backup
exit 1
end
tar_file = if ENV['BACKUP'].present?
File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
tar_file = if source_backup_id.present?
File.basename(source_backup_id) + FILE_NAME_SUFFIX
else
backup_file_list.first
end
@ -501,12 +513,19 @@ module Backup
@tar_file ||= "#{backup_id}#{FILE_NAME_SUFFIX}"
end
def full_backup_id
full_backup_id = backup_information[:full_backup_id]
full_backup_id ||= File.basename(ENV['PREVIOUS_BACKUP']) if ENV['PREVIOUS_BACKUP'].present?
full_backup_id ||= backup_id
full_backup_id
end
def backup_id
@backup_id ||= if ENV['BACKUP'].present?
File.basename(ENV['BACKUP'])
else
"#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}"
end
if ENV['BACKUP'].present?
File.basename(ENV['BACKUP'])
else
"#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}"
end
end
def create_attributes

View file

@ -5543,9 +5543,21 @@ msgstr ""
msgid "BackgroundMigrations|Background migrations are used to perform data migrations whenever a migration exceeds the time limits in our guidelines. %{linkStart}Learn more%{linkEnd}"
msgstr ""
msgid "BackgroundMigrations|Batch size"
msgstr ""
msgid "BackgroundMigrations|Database"
msgstr ""
msgid "BackgroundMigrations|Failed jobs:"
msgstr ""
msgid "BackgroundMigrations|Finished at"
msgstr ""
msgid "BackgroundMigrations|Started at"
msgstr ""
msgid "Badges"
msgstr ""
@ -44407,9 +44419,6 @@ msgstr ""
msgid "entries cannot contain HTML tags"
msgstr ""
msgid "environment_id parameter is required when type is container_policy"
msgstr ""
msgid "epic"
msgstr ""

View file

@ -16,7 +16,7 @@ module QA
initial_wiki.project.remove_via_api!
end
it 'creates a formatted Wiki page with an image uploaded', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/361545', type: :investigating }, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347640' do
it 'creates a formatted Wiki page with an image uploaded', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347640' do
initial_wiki.visit!
Page::Project::Wiki::Show.perform(&:click_new_page)

View file

@ -430,12 +430,6 @@ RSpec.describe "Admin Runners" do
end
context "when visiting outdated URLs" do
it 'updates NOT_CONNECTED runner status to NEVER_CONNECTED' do
visit admin_runners_path('status[]': 'NOT_CONNECTED')
expect(page).to have_current_path(admin_runners_path('status[]': 'NEVER_CONTACTED') )
end
it 'updates ACTIVE runner status to paused=false' do
visit admin_runners_path('status[]': 'ACTIVE')

View file

@ -42,6 +42,28 @@ RSpec.describe "Admin > Admin sees background migrations" do
end
end
it 'can view failed jobs' do
visit admin_background_migration_path(failed_migration)
within '#content-body' do
expect(page).to have_content('Failed jobs')
expect(page).to have_content('Id')
expect(page).to have_content('Started at')
expect(page).to have_content('Finished at')
expect(page).to have_content('Batch size')
end
end
context 'when there are no failed jobs' do
it 'dos not display failed jobs' do
visit admin_background_migration_path(active_migration)
within '#content-body' do
expect(page).not_to have_content('Failed jobs')
end
end
end
it 'can view queued migrations and pause and resume them' do
visit admin_background_migrations_path

View file

@ -147,6 +147,7 @@ RSpec.describe Backup::Manager do
let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz task2.tar.gz} }
let(:backup_time) { Time.utc(2019, 1, 1) }
let(:backup_id) { "1546300800_2019_01_01_#{Gitlab::VERSION}" }
let(:full_backup_id) { backup_id }
let(:pack_tar_file) { "#{backup_id}_gitlab_backup.tar" }
let(:pack_tar_system_options) { { out: [pack_tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
let(:pack_tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, pack_tar_system_options] }
@ -166,8 +167,8 @@ RSpec.describe Backup::Manager do
allow(Gitlab::BackupLogger).to receive(:info)
allow(Kernel).to receive(:system).and_return(true)
allow(task1).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz'), backup_id)
allow(task2).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz'), backup_id)
allow(task1).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz'), full_backup_id)
allow(task2).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz'), full_backup_id)
end
it 'executes tar' do
@ -588,6 +589,14 @@ RSpec.describe Backup::Manager do
end
expect(Kernel).not_to have_received(:system).with(*pack_tar_cmdline)
expect(YAML.load_file(File.join(Gitlab.config.backup.path, 'backup_information.yml'))).to include(
backup_created_at: backup_time.localtime,
db_version: be_a(String),
gitlab_version: Gitlab::VERSION,
installation_type: Gitlab::INSTALLATION_TYPE,
skipped: 'tar',
tar_version: be_a(String)
)
end
end
@ -595,11 +604,11 @@ RSpec.describe Backup::Manager do
let(:incremental_env) { 'true' }
let(:gitlab_version) { Gitlab::VERSION }
let(:backup_id) { "1546300800_2019_01_01_#{gitlab_version}" }
let(:pack_tar_file) { "#{backup_id}_gitlab_backup.tar" }
let(:unpack_tar_cmdline) { ['tar', '-xf', pack_tar_file] }
let(:unpack_tar_file) { "#{full_backup_id}_gitlab_backup.tar" }
let(:unpack_tar_cmdline) { ['tar', '-xf', unpack_tar_file] }
let(:backup_information) do
{
backup_created_at: backup_time,
backup_created_at: Time.zone.parse('2018-01-01'),
gitlab_version: gitlab_version
}
end
@ -682,7 +691,9 @@ RSpec.describe Backup::Manager do
end
it 'unpacks and packs the backup' do
subject.create # rubocop:disable Rails/SaveBang
travel_to(backup_time) do
subject.create # rubocop:disable Rails/SaveBang
end
expect(Kernel).to have_received(:system).with(*unpack_tar_cmdline)
expect(Kernel).to have_received(:system).with(*pack_tar_cmdline)
@ -732,21 +743,137 @@ RSpec.describe Backup::Manager do
end
end
context 'when PREVIOUS_BACKUP variable is set to a non-existing file' do
before do
allow(Dir).to receive(:glob).and_return(
[
'1451606400_2016_01_01_gitlab_backup.tar'
]
)
allow(File).to receive(:exist?).and_return(false)
stub_env('PREVIOUS_BACKUP', 'wrong')
end
it 'fails the operation and prints an error' do
expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
expect(File).to have_received(:exist?).with('wrong_gitlab_backup.tar')
expect(progress).to have_received(:puts)
.with(a_string_matching('The backup file wrong_gitlab_backup.tar does not exist'))
end
end
context 'when PREVIOUS_BACKUP variable is set to a correct file' do
let(:full_backup_id) { 'some_previous_backup' }
before do
allow(Gitlab::BackupLogger).to receive(:info)
allow(Dir).to receive(:glob).and_return(
[
'some_previous_backup_gitlab_backup.tar'
]
)
allow(File).to receive(:exist?).with('some_previous_backup_gitlab_backup.tar').and_return(true)
allow(Kernel).to receive(:system).and_return(true)
stub_env('PREVIOUS_BACKUP', '/ignored/path/some_previous_backup')
end
it 'unpacks and packs the backup' do
travel_to(backup_time) do
subject.create # rubocop:disable Rails/SaveBang
end
expect(Kernel).to have_received(:system).with(*unpack_tar_cmdline)
expect(Kernel).to have_received(:system).with(*pack_tar_cmdline)
end
context 'untar fails' do
before do
expect(Kernel).to receive(:system).with(*unpack_tar_cmdline).and_return(false)
end
it 'logs a failure' do
expect do
travel_to(backup_time) do
subject.create # rubocop:disable Rails/SaveBang
end
end.to raise_error(SystemExit)
expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Unpacking backup failed')
end
end
context 'tar fails' do
before do
expect(Kernel).to receive(:system).with(*pack_tar_cmdline).and_return(false)
end
it 'logs a failure' do
expect do
travel_to(backup_time) do
subject.create # rubocop:disable Rails/SaveBang
end
end.to raise_error(Backup::Error, 'Backup failed')
expect(Gitlab::BackupLogger).to have_received(:info).with(message: "Creating archive #{pack_tar_file} failed")
end
end
context 'on version mismatch' do
let(:backup_information) do
{
backup_created_at: Time.zone.parse('2018-01-01'),
gitlab_version: "not #{gitlab_version}"
}
end
it 'stops the process' do
expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
expect(progress).to have_received(:puts)
.with(a_string_matching('GitLab version mismatch'))
end
end
end
context 'when there is a non-tarred backup in the directory' do
let(:full_backup_id) { "1514764800_2018_01_01_#{Gitlab::VERSION}" }
let(:backup_information) do
{
backup_created_at: Time.zone.parse('2018-01-01'),
gitlab_version: gitlab_version,
skipped: 'tar'
}
end
before do
allow(Dir).to receive(:glob).and_return(
[
'backup_information.yml'
]
)
allow(File).to receive(:exist?).and_return(true)
allow(File).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'backup_information.yml')).and_return(true)
end
it 'selects the non-tarred backup to restore from' do
subject.create # rubocop:disable Rails/SaveBang
after do
FileUtils.rm(File.join(Gitlab.config.backup.path, 'backup_information.yml'), force: true)
end
it 'updates the non-tarred backup' do
travel_to(backup_time) do
subject.create # rubocop:disable Rails/SaveBang
end
expect(progress).to have_received(:puts)
.with(a_string_matching('Non tarred backup found '))
expect(progress).to have_received(:puts)
.with(a_string_matching("Backup #{backup_id} is done"))
expect(YAML.load_file(File.join(Gitlab.config.backup.path, 'backup_information.yml'))).to include(
backup_created_at: backup_time,
full_backup_id: full_backup_id,
gitlab_version: Gitlab::VERSION,
skipped: 'tar'
)
end
context 'on version mismatch' do

View file

@ -27,6 +27,17 @@ RSpec.describe Gitlab::Ci::Ansi2json do
])
end
it 'ignores empty newlines' do
expect(convert_json("Hello\n\nworld")).to eq([
{ offset: 0, content: [{ text: 'Hello' }] },
{ offset: 7, content: [{ text: 'world' }] }
])
expect(convert_json("Hello\r\n\r\nworld")).to eq([
{ offset: 0, content: [{ text: 'Hello' }] },
{ offset: 9, content: [{ text: 'world' }] }
])
end
it 'replace the current line when encountering \r' do
expect(convert_json("Hello\rworld")).to eq([
{ offset: 0, content: [{ text: 'world' }] }

View file

@ -10,12 +10,21 @@ RSpec.describe Admin::BackgroundMigrationsController, :enable_admin_mode do
end
describe 'GET #show' do
let(:migration) { create(:batched_background_migration) }
context 'when the migration is valid' do
let(:migration) { create(:batched_background_migration) }
let!(:failed_job) { create(:batched_background_migration_job, :failed, batched_migration: migration) }
it 'fetches the migration' do
get admin_background_migration_path(migration)
it 'fetches the migration' do
get admin_background_migration_path(migration)
expect(response).to have_gitlab_http_status(:ok)
expect(response).to have_gitlab_http_status(:ok)
end
it 'returns failed jobs' do
get admin_background_migration_path(migration)
expect(assigns(:failed_jobs)).to match_array([failed_job])
end
end
context 'when the migration does not exist' do