From eb1a3798adb3836e4ebe641c8eb962b6f7220004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 9 Aug 2018 01:08:09 -0400 Subject: [PATCH] Remove storage path dependency of gitaly install task --- changelogs/unreleased/gitaly-install-path.yml | 5 ++ doc/install/installation.md | 6 +- lib/gitlab/setup_helper.rb | 13 ++-- lib/tasks/gitlab/gitaly.rake | 24 ++----- spec/support/helpers/test_env.rb | 12 +++- spec/tasks/gitlab/gitaly_rake_spec.rb | 67 +++++-------------- 6 files changed, 45 insertions(+), 82 deletions(-) create mode 100644 changelogs/unreleased/gitaly-install-path.yml diff --git a/changelogs/unreleased/gitaly-install-path.yml b/changelogs/unreleased/gitaly-install-path.yml new file mode 100644 index 00000000000..4b24cd81dc7 --- /dev/null +++ b/changelogs/unreleased/gitaly-install-path.yml @@ -0,0 +1,5 @@ +--- +title: Remove storage path dependency of gitaly install task +merge_request: 21101 +author: +type: changed diff --git a/doc/install/installation.md b/doc/install/installation.md index 8c7f80fd8e8..0351d67d1d5 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -153,7 +153,7 @@ page](https://golang.org/dl). # Remove former Go installation folder sudo rm -rf /usr/local/go - + curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz @@ -494,11 +494,11 @@ Make GitLab start on boot: ### Install Gitaly # Fetch Gitaly source with Git and compile with Go - sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production + sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories]" RAILS_ENV=production You can specify a different Git repository by providing it as an extra parameter: - sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,https://example.com/gitaly.git]" RAILS_ENV=production + sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly,/home/git/repositories,https://example.com/gitaly.git]" RAILS_ENV=production Next, make sure gitaly configured: diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index b2d75aac1d0..5b68e4470cd 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -1,3 +1,5 @@ +require 'toml-rb' + module Gitlab module SetupHelper class << self @@ -9,7 +11,7 @@ module Gitlab # because it uses a Unix socket. # For development and testing purposes, an extra storage is added to gitaly, # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true) storages = [] address = nil @@ -24,10 +26,7 @@ module Gitlab address = val['gitaly_address'] end - # https://gitlab.com/gitlab-org/gitaly/issues/1238 - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - storages << { name: key, path: val.legacy_disk_path } - end + storages << { name: key, path: storage_paths[key] } end if Rails.env.test? @@ -44,12 +43,12 @@ module Gitlab end # rubocop:disable Rails/Output - def create_gitaly_configuration(dir, force: false) + def create_gitaly_configuration(dir, storage_paths, force: false) config_path = File.join(dir, 'config.toml') FileUtils.rm_f(config_path) if force File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml(dir) + f.puts gitaly_configuration_toml(dir, storage_paths) end rescue Errno::EEXIST puts "Skipping config.toml generation:" diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index e9ca6404fe8..80de3d2ef51 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,13 +1,12 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir, :repo] => :gitlab_environment do |t, args| - require 'toml-rb' - + task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab - unless args.dir.present? - abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") + unless args.dir.present? && args.storage_path.present? + abort %(Please specify the directory where you want to install gitaly and the path for the default storage +Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") end args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') @@ -27,7 +26,8 @@ namespace :gitlab do "BUNDLE_PATH=#{Bundler.bundle_path}") end - Gitlab::SetupHelper.create_gitaly_configuration(args.dir) + storage_paths = { 'default' => args.storage_path } + Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths) Dir.chdir(args.dir) do # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? @@ -35,17 +35,5 @@ namespace :gitlab do end end end - - desc "GitLab | Print storage configuration in TOML format" - task storage_config: :environment do - require 'toml-rb' - - puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" - puts "# This is in TOML format suitable for use in Gitaly's config.toml file." - - # Exclude gitaly-ruby configuration because that depends on the gitaly - # installation directory. - puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) - end end end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index e531495d917..34318fd0b05 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -66,6 +66,7 @@ module TestEnv TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**') REPOS_STORAGE = 'default'.freeze + BROKEN_STORAGE = 'broken'.freeze # Test environment # @@ -156,10 +157,11 @@ module TestEnv component_timed_setup('Gitaly', install_dir: gitaly_dir, version: Gitlab::GitalyClient.expected_server_version, - task: "gitlab:gitaly:install[#{gitaly_dir}]") do + task: "gitlab:gitaly:install[#{gitaly_dir},#{repos_path}]") do - # Always re-create config, in case it's outdated. This is fast anyway. - Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true) + # Re-create config, to specify the broken storage path + storage_paths = { 'default' => repos_path, 'broken' => broken_path } + Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, storage_paths, force: true) start_gitaly(gitaly_dir) end @@ -247,6 +249,10 @@ module TestEnv @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path end + def broken_path + @broken_path ||= Gitlab.config.repositories.storages[BROKEN_STORAGE].legacy_disk_path + end + def backup_path Gitlab.config.backup.path end diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 4545226d78c..e6e4d9504d9 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do describe 'install' do let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' } let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s } + let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s } let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp } + subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) } + context 'no dir given' do it 'aborts and display a help message' do # avoid writing task output to spec progress allow($stderr).to receive :write - expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/ + expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ + end + end + + context 'no storage path given' do + it 'aborts and display a help message' do + allow($stderr).to receive :write + expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ end end @@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).and_raise 'Git error' - expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error' + expect { subject }.to raise_error 'Git error' end end @@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory' do expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end context 'when Rails.env is test' do @@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory with --no-deployment flag for bundle' do expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end end end end - - describe 'storage_config' do - it 'prints storage configuration in a TOML format' do - config = { - 'default' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/default', - 'gitaly_address' => 'unix:/path/to/my.socket' - ), - 'nfs_01' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/nfs_01', - 'gitaly_address' => 'unix:/path/to/my.socket' - ) - } - allow(Gitlab.config.repositories).to receive(:storages).and_return(config) - allow(Rails.env).to receive(:test?).and_return(false) - - expected_output = '' - Timecop.freeze do - expected_output = <<~TOML - # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)} - # This is in TOML format suitable for use in Gitaly's config.toml file. - bin_dir = "tmp/tests/gitaly" - socket_path = "/path/to/my.socket" - [gitlab-shell] - dir = "#{Gitlab.config.gitlab_shell.path}" - [[storage]] - name = "default" - path = "/path/to/default" - [[storage]] - name = "nfs_01" - path = "/path/to/nfs_01" - TOML - end - - expect { run_rake_task('gitlab:gitaly:storage_config')} - .to output(expected_output).to_stdout - - parsed_output = TomlRB.parse(expected_output) - config.each do |name, params| - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) - end - end - end - end end