kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Remove storage path dependency of gitaly install task

Browse source
This commit is contained in:
Alejandro Rodríguez 2018-08-09 01:08:09 -04:00
parent 93c7b6c51a
commit eb1a3798ad
6 changed files with 45 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
changelogs/unreleased/gitaly-install-path.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Remove storage path dependency of gitaly install task
merge_request: 21101
author:
type: changed

6
doc/install/installation.md
View file

@ -153,7 +153,7 @@ page](https://golang.org/dl).
# Remove former Go installation folder
sudo rm -rf /usr/local/go
curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz
echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \
sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz
@ -494,11 +494,11 @@ Make GitLab start on boot:
### Install Gitaly
# Fetch Gitaly source with Git and compile with Go
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories]" RAILS_ENV=production
You can specify a different Git repository by providing it as an extra parameter:
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,https://example.com/gitaly.git]" RAILS_ENV=production
sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories,https://example.com/gitaly.git]" RAILS_ENV=production
Next, make sure gitaly configured:

13
lib/gitlab/setup_helper.rb
View file

@ -1,3 +1,5 @@
require 'toml-rb'
module Gitlab
module SetupHelper
class << self
@ -9,7 +11,7 @@ module Gitlab
# because it uses a Unix socket.
# For development and testing purposes, an extra storage is added to gitaly,
# which is not known to Rails, but must be explicitly stubbed.
def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true)
def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true)
storages = []
address = nil
@ -24,10 +26,7 @@ module Gitlab
address = val['gitaly_address']
end
# https://gitlab.com/gitlab-org/gitaly/issues/1238
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
storages << { name: key, path: val.legacy_disk_path }
end
storages << { name: key, path: storage_paths[key] }
end
if Rails.env.test?
@ -44,12 +43,12 @@ module Gitlab
end
# rubocop:disable Rails/Output
def create_gitaly_configuration(dir, force: false)
def create_gitaly_configuration(dir, storage_paths, force: false)
config_path = File.join(dir, 'config.toml')
FileUtils.rm_f(config_path) if force
File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f|
f.puts gitaly_configuration_toml(dir)
f.puts gitaly_configuration_toml(dir, storage_paths)
end
rescue Errno::EEXIST
puts "Skipping config.toml generation:"

24
lib/tasks/gitlab/gitaly.rake
View file

@ -1,13 +1,12 @@
namespace :gitlab do
namespace :gitaly do
desc "GitLab | Install or upgrade gitaly"
task :install, [:dir, :repo] => :gitlab_environment do |t, args|
require 'toml-rb'
task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
warn_user_is_not_gitlab
unless args.dir.present?
abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]")
unless args.dir.present? && args.storage_path.present?
abort %(Please specify the directory where you want to install gitaly and the path for the default storage
Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
end
args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
@ -27,7 +26,8 @@ namespace :gitlab do
"BUNDLE_PATH=#{Bundler.bundle_path}")
end
Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
storage_paths = { 'default' => args.storage_path }
Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths)
Dir.chdir(args.dir) do
# In CI we run scripts/gitaly-test-build instead of this command
unless ENV['CI'].present?
@ -35,17 +35,5 @@ namespace :gitlab do
end
end
end
desc "GitLab | Print storage configuration in TOML format"
task storage_config: :environment do
require 'toml-rb'
puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
# Exclude gitaly-ruby configuration because that depends on the gitaly
# installation directory.
puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
end
end
end

12
spec/support/helpers/test_env.rb
View file

@ -66,6 +66,7 @@ module TestEnv
TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**')
REPOS_STORAGE = 'default'.freeze
BROKEN_STORAGE = 'broken'.freeze
# Test environment
#
@ -156,10 +157,11 @@ module TestEnv
component_timed_setup('Gitaly',
install_dir: gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:install[#{gitaly_dir}]") do
task: "gitlab:gitaly:install[#{gitaly_dir},#{repos_path}]") do
# Always re-create config, in case it's outdated. This is fast anyway.
Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true)
# Re-create config, to specify the broken storage path
storage_paths = { 'default' => repos_path, 'broken' => broken_path }
Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, storage_paths, force: true)
start_gitaly(gitaly_dir)
end
@ -247,6 +249,10 @@ module TestEnv
@repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
end
def broken_path
@broken_path ||= Gitlab.config.repositories.storages[BROKEN_STORAGE].legacy_disk_path
end
def backup_path
Gitlab.config.backup.path
end

67
spec/tasks/gitlab/gitaly_rake_spec.rb
View file

@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do
describe 'install' do
let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
context 'no dir given' do
it 'aborts and display a help message' do
# avoid writing task output to spec progress
allow($stderr).to receive :write
expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/
expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
end
end
context 'no storage path given' do
it 'aborts and display a help message' do
allow($stderr).to receive :write
expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
end
end
@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(main_object)
.to receive(:checkout_or_clone_version).and_raise 'Git error'
expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error'
expect { subject }.to raise_error 'Git error'
end
end
@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(main_object)
.to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
run_rake_task('gitlab:gitaly:install', clone_path)
subject
end
end
@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do
expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path)
subject
end
end
@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do
it 'calls make in the gitaly directory' do
expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path)
subject
end
context 'when Rails.env is test' do
@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do
it 'calls make in the gitaly directory with --no-deployment flag for bundle' do
expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true)
run_rake_task('gitlab:gitaly:install', clone_path)
subject
end
end
end
end
end
describe 'storage_config' do
it 'prints storage configuration in a TOML format' do
config = {
'default' => Gitlab::GitalyClient::StorageSettings.new(
'path' => '/path/to/default',
'gitaly_address' => 'unix:/path/to/my.socket'
),
'nfs_01' => Gitlab::GitalyClient::StorageSettings.new(
'path' => '/path/to/nfs_01',
'gitaly_address' => 'unix:/path/to/my.socket'
)
}
allow(Gitlab.config.repositories).to receive(:storages).and_return(config)
allow(Rails.env).to receive(:test?).and_return(false)
expected_output = ''
Timecop.freeze do
expected_output = <<~TOML
# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}
# This is in TOML format suitable for use in Gitaly's config.toml file.
bin_dir = "tmp/tests/gitaly"
socket_path = "/path/to/my.socket"
[gitlab-shell]
dir = "#{Gitlab.config.gitlab_shell.path}"
[[storage]]
name = "default"
path = "/path/to/default"
[[storage]]
name = "nfs_01"
path = "/path/to/nfs_01"
TOML
end
expect { run_rake_task('gitlab:gitaly:storage_config')}
.to output(expected_output).to_stdout
parsed_output = TomlRB.parse(expected_output)
config.each do |name, params|
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path })
end
end
end
end
end