Merge branch 'feature/migrate-import-repository-to-gitaly' into 'master'
Migrate importing repository to Gitaly Closes gitaly#907 See merge request gitlab-org/gitlab-ce!16431
This commit is contained in:
commit
15ef5d6694
2
Gemfile
2
Gemfile
|
@ -403,7 +403,7 @@ group :ed25519 do
|
|||
end
|
||||
|
||||
# Gitaly GRPC client
|
||||
gem 'gitaly-proto', '~> 0.69.0', require: 'gitaly'
|
||||
gem 'gitaly-proto', '~> 0.73.0', require: 'gitaly'
|
||||
|
||||
gem 'toml-rb', '~> 0.3.15', require: false
|
||||
|
||||
|
|
|
@ -284,7 +284,7 @@ GEM
|
|||
po_to_json (>= 1.0.0)
|
||||
rails (>= 3.2.0)
|
||||
gherkin-ruby (0.3.2)
|
||||
gitaly-proto (0.69.0)
|
||||
gitaly-proto (0.73.0)
|
||||
google-protobuf (~> 3.1)
|
||||
grpc (~> 1.0)
|
||||
github-linguist (4.7.6)
|
||||
|
@ -1054,7 +1054,7 @@ DEPENDENCIES
|
|||
gettext (~> 3.2.2)
|
||||
gettext_i18n_rails (~> 1.8.0)
|
||||
gettext_i18n_rails_js (~> 1.2.0)
|
||||
gitaly-proto (~> 0.69.0)
|
||||
gitaly-proto (~> 0.73.0)
|
||||
github-linguist (~> 4.7.0)
|
||||
gitlab-flowdock-git-hook (~> 1.0.1)
|
||||
gitlab-markup (~> 1.6.2)
|
||||
|
|
|
@ -44,29 +44,13 @@ module Gitlab
|
|||
# Import project via git clone --bare
|
||||
# URL must be publicly cloneable
|
||||
def import_project(source, timeout)
|
||||
# Skip import if repo already exists
|
||||
return false if File.exist?(repository_absolute_path)
|
||||
|
||||
masked_source = mask_password_in_url(source)
|
||||
|
||||
logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
|
||||
cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
|
||||
|
||||
success = run_with_timeout(cmd, timeout, nil)
|
||||
|
||||
unless success
|
||||
logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
|
||||
FileUtils.rm_rf(repository_absolute_path)
|
||||
return false
|
||||
Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
|
||||
if is_enabled
|
||||
gitaly_import_repository(source)
|
||||
else
|
||||
git_import_repository(source, timeout)
|
||||
end
|
||||
end
|
||||
|
||||
Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
|
||||
|
||||
# The project was imported successfully.
|
||||
# Remove the origin URL since it may contain password.
|
||||
remove_origin_in_repo
|
||||
|
||||
true
|
||||
end
|
||||
|
||||
def fork_repository(new_shard_path, new_repository_relative_path)
|
||||
|
@ -231,6 +215,42 @@ module Gitlab
|
|||
raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
|
||||
end
|
||||
|
||||
def git_import_repository(source, timeout)
|
||||
# Skip import if repo already exists
|
||||
return false if File.exist?(repository_absolute_path)
|
||||
|
||||
masked_source = mask_password_in_url(source)
|
||||
|
||||
logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
|
||||
cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
|
||||
|
||||
success = run_with_timeout(cmd, timeout, nil)
|
||||
|
||||
unless success
|
||||
logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
|
||||
FileUtils.rm_rf(repository_absolute_path)
|
||||
return false
|
||||
end
|
||||
|
||||
Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
|
||||
|
||||
# The project was imported successfully.
|
||||
# Remove the origin URL since it may contain password.
|
||||
remove_origin_in_repo
|
||||
|
||||
true
|
||||
end
|
||||
|
||||
def gitaly_import_repository(source)
|
||||
raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
|
||||
|
||||
Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
|
||||
true
|
||||
rescue GRPC::BadStatus => e
|
||||
@output << e.message
|
||||
false
|
||||
end
|
||||
|
||||
def git_fork_repository(new_shard_path, new_repository_relative_path)
|
||||
from_path = repository_absolute_path
|
||||
to_path = File.join(new_shard_path, new_repository_relative_path)
|
||||
|
|
|
@ -100,6 +100,21 @@ module Gitlab
|
|||
)
|
||||
end
|
||||
|
||||
def import_repository(source)
|
||||
request = Gitaly::CreateRepositoryFromURLRequest.new(
|
||||
repository: @gitaly_repo,
|
||||
url: source
|
||||
)
|
||||
|
||||
GitalyClient.call(
|
||||
@storage,
|
||||
:repository_service,
|
||||
:create_repository_from_url,
|
||||
request,
|
||||
timeout: GitalyClient.default_timeout
|
||||
)
|
||||
end
|
||||
|
||||
def rebase_in_progress?(rebase_id)
|
||||
request = Gitaly::IsRebaseInProgressRequest.new(
|
||||
repository: @gitaly_repo,
|
||||
|
|
|
@ -158,39 +158,55 @@ describe Gitlab::Git::GitlabProjects do
|
|||
|
||||
subject { gl_projects.import_project(import_url, timeout) }
|
||||
|
||||
context 'success import' do
|
||||
it 'imports a repo' do
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
|
||||
shared_examples 'importing repository' do
|
||||
context 'success import' do
|
||||
it 'imports a repo' do
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
|
||||
|
||||
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
|
||||
expect(logger).to receive(:info).with(message)
|
||||
is_expected.to be_truthy
|
||||
|
||||
is_expected.to be_truthy
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
|
||||
end
|
||||
end
|
||||
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
|
||||
context 'already exists' do
|
||||
it "doesn't import" do
|
||||
FileUtils.mkdir_p(tmp_repo_path)
|
||||
|
||||
is_expected.to be_falsy
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'already exists' do
|
||||
it "doesn't import" do
|
||||
FileUtils.mkdir_p(tmp_repo_path)
|
||||
|
||||
is_expected.to be_falsy
|
||||
end
|
||||
context 'when Gitaly import_repository feature is enabled' do
|
||||
it_behaves_like 'importing repository'
|
||||
end
|
||||
|
||||
context 'timeout' do
|
||||
it 'does not import a repo' do
|
||||
stub_spawn_timeout(cmd, timeout, nil)
|
||||
context 'when Gitaly import_repository feature is disabled', :disable_gitaly do
|
||||
describe 'logging' do
|
||||
it 'imports a repo' do
|
||||
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
|
||||
expect(logger).to receive(:info).with(message)
|
||||
|
||||
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
|
||||
expect(logger).to receive(:error).with(message)
|
||||
|
||||
is_expected.to be_falsy
|
||||
|
||||
expect(gl_projects.output).to eq("Timed out\n")
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
|
||||
subject
|
||||
end
|
||||
end
|
||||
|
||||
context 'timeout' do
|
||||
it 'does not import a repo' do
|
||||
stub_spawn_timeout(cmd, timeout, nil)
|
||||
|
||||
message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
|
||||
expect(logger).to receive(:error).with(message)
|
||||
|
||||
is_expected.to be_falsy
|
||||
|
||||
expect(gl_projects.output).to eq("Timed out\n")
|
||||
expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
|
||||
end
|
||||
end
|
||||
|
||||
it_behaves_like 'importing repository'
|
||||
end
|
||||
end
|
||||
|
||||
|
|
Loading…
Reference in New Issue