2019-04-11 12:17:24 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2018-05-03 12:55:14 +00:00
|
|
|
require 'spec_helper'
|
|
|
|
|
|
|
|
describe Projects::UpdateRemoteMirrorService do
|
2018-09-10 19:12:49 +00:00
|
|
|
let(:project) { create(:project, :repository) }
|
2018-05-03 12:55:14 +00:00
|
|
|
let(:remote_project) { create(:forked_project_with_submodules) }
|
|
|
|
let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
|
2018-09-10 19:12:49 +00:00
|
|
|
let(:remote_name) { remote_mirror.remote_name }
|
2018-05-03 12:55:14 +00:00
|
|
|
|
2018-09-10 19:12:49 +00:00
|
|
|
subject(:service) { described_class.new(project, project.creator) }
|
2018-05-03 12:55:14 +00:00
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
describe '#execute' do
|
|
|
|
subject(:execute!) { service.execute(remote_mirror, 0) }
|
|
|
|
|
2018-05-03 12:55:14 +00:00
|
|
|
before do
|
2018-09-10 19:12:49 +00:00
|
|
|
project.repository.add_branch(project.owner, 'existing-branch', 'master')
|
2018-06-25 09:37:24 +00:00
|
|
|
|
|
|
|
allow(remote_mirror).to receive(:update_repository).and_return(true)
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
it 'ensures the remote exists' do
|
2018-11-12 10:52:48 +00:00
|
|
|
stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
|
2018-09-10 19:12:49 +00:00
|
|
|
|
|
|
|
expect(remote_mirror).to receive(:ensure_remote!)
|
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
execute!
|
2018-09-10 19:12:49 +00:00
|
|
|
end
|
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
it 'fetches the remote repository' do
|
2018-09-10 19:12:49 +00:00
|
|
|
expect(project.repository)
|
|
|
|
.to receive(:fetch_remote)
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
.with(remote_mirror.remote_name, no_tags: true, ssh_auth: remote_mirror)
|
2018-09-10 19:12:49 +00:00
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
execute!
|
2018-09-10 19:12:49 +00:00
|
|
|
end
|
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
it 'marks the mirror as started when beginning' do
|
|
|
|
expect(remote_mirror).to receive(:update_start!).and_call_original
|
|
|
|
|
|
|
|
execute!
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'marks the mirror as successfully finished' do
|
2018-11-12 10:52:48 +00:00
|
|
|
stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
|
2018-09-10 19:12:49 +00:00
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
result = execute!
|
2018-05-03 12:55:14 +00:00
|
|
|
|
|
|
|
expect(result[:status]).to eq(:success)
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
expect(remote_mirror).to be_finished
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'marks the mirror as failed and raises the error when an unexpected error occurs' do
|
|
|
|
allow(project.repository).to receive(:fetch_remote).and_raise('Badly broken')
|
|
|
|
|
|
|
|
expect { execute! }.to raise_error /Badly broken/
|
|
|
|
|
|
|
|
expect(remote_mirror).to be_failed
|
|
|
|
expect(remote_mirror.last_error).to include('Badly broken')
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when the update fails because of a `Gitlab::Git::CommandError`' do
|
|
|
|
before do
|
|
|
|
allow(project.repository).to receive(:fetch_remote).and_raise(Gitlab::Git::CommandError.new('fetch failed'))
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'wraps `Gitlab::Git::CommandError`s in a service error' do
|
|
|
|
expect(execute!).to eq(status: :error, message: 'fetch failed')
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'marks the mirror as to be retried' do
|
|
|
|
execute!
|
|
|
|
|
|
|
|
expect(remote_mirror).to be_to_retry
|
|
|
|
expect(remote_mirror.last_error).to include('fetch failed')
|
|
|
|
end
|
|
|
|
|
|
|
|
it "marks the mirror as failed after #{described_class::MAX_TRIES} tries" do
|
|
|
|
service.execute(remote_mirror, described_class::MAX_TRIES)
|
|
|
|
|
|
|
|
expect(remote_mirror).to be_failed
|
|
|
|
expect(remote_mirror.last_error).to include('fetch failed')
|
|
|
|
end
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
|
2018-06-25 09:37:24 +00:00
|
|
|
context 'when syncing all branches' do
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
it 'push all the branches the first time' do
|
2018-11-12 10:52:48 +00:00
|
|
|
stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
|
2018-05-03 12:55:14 +00:00
|
|
|
|
2018-06-25 09:37:24 +00:00
|
|
|
expect(remote_mirror).to receive(:update_repository).with({})
|
2018-05-03 12:55:14 +00:00
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
execute!
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-06-25 09:37:24 +00:00
|
|
|
context 'when only syncing protected branches' do
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
it 'sync updated protected branches' do
|
2018-11-12 10:52:48 +00:00
|
|
|
stub_fetch_remote(project, remote_name: remote_name, ssh_auth: remote_mirror)
|
2018-09-10 19:12:49 +00:00
|
|
|
protected_branch = create_protected_branch(project)
|
2018-06-25 09:37:24 +00:00
|
|
|
remote_mirror.only_protected_branches = true
|
2018-05-03 12:55:14 +00:00
|
|
|
|
2018-09-10 19:12:49 +00:00
|
|
|
expect(remote_mirror)
|
|
|
|
.to receive(:update_repository)
|
|
|
|
.with(only_branches_matching: [protected_branch.name])
|
2018-05-03 12:55:14 +00:00
|
|
|
|
Rework retry strategy for remote mirrors
**Prevention of running 2 simultaneous updates**
Instead of using `RemoteMirror#update_status` and raise an error if
it's already running to prevent the same mirror being updated at the
same time we now use `Gitlab::ExclusiveLease` for that.
When we fail to obtain a lease in 3 tries, 30 seconds apart, we bail
and reschedule. We'll reschedule faster for the protected branches.
If the mirror already ran since it was scheduled, the job will be
skipped.
**Error handling: Remote side**
When an update fails because of a `Gitlab::Git::CommandError`, we
won't track this error in sentry, this could be on the remote side:
for example when branches have diverged.
In this case, we'll try 3 times scheduled 1 or 5 minutes apart.
In between, the mirror is marked as "to_retry", the error would be
visible to the user when they visit the settings page.
After 3 tries we'll mark the mirror as failed and notify the user.
We won't track this error in sentry, as it's not likely we can help
it.
The next event that would trigger a new refresh.
**Error handling: our side**
If an unexpected error occurs, we mark the mirror as failed, but we'd
still retry the job based on the regular sidekiq retries with
backoff. Same as we used to
The error would be reported in sentry, since its likely we need to do
something about it.
2019-08-13 20:52:01 +00:00
|
|
|
execute!
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
|
2018-09-10 19:12:49 +00:00
|
|
|
def create_protected_branch(project)
|
|
|
|
branch_name = project.repository.branch_names.find { |n| n != 'existing-branch' }
|
|
|
|
create(:protected_branch, project: project, name: branch_name)
|
|
|
|
end
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-11-12 10:52:48 +00:00
|
|
|
def stub_fetch_remote(project, remote_name:, ssh_auth:)
|
2018-09-10 19:12:49 +00:00
|
|
|
allow(project.repository)
|
|
|
|
.to receive(:fetch_remote)
|
2018-11-12 10:52:48 +00:00
|
|
|
.with(remote_name, no_tags: true, ssh_auth: ssh_auth) { fetch_remote(project.repository, remote_name) }
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
|
2018-09-10 19:12:49 +00:00
|
|
|
def fetch_remote(repository, remote_name)
|
|
|
|
local_branch_names(repository).each do |branch|
|
|
|
|
commit = repository.commit(branch)
|
|
|
|
repository.write_ref("refs/remotes/#{remote_name}/#{branch}", commit.id) if commit
|
2018-05-03 12:55:14 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-09-10 19:12:49 +00:00
|
|
|
def local_branch_names(repository)
|
2018-05-03 12:55:14 +00:00
|
|
|
branch_names = repository.branches.map(&:name)
|
|
|
|
# we want the protected branch to be pushed first
|
|
|
|
branch_names.unshift(branch_names.delete('master'))
|
|
|
|
end
|
|
|
|
end
|