From fbbf177e3b604bebce3b10f8eea8920ff5606fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 28 Sep 2016 12:45:46 +0200 Subject: [PATCH 1/3] New `gitlab:workhorse:install` rake task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- ...e-task-similar-to-gitlab-shell-install.yml | 4 + doc/install/installation.md | 28 +++-- doc/update/8.12-to-8.13.md | 2 +- lib/tasks/gitlab/shell.rake | 42 +------ lib/tasks/gitlab/task_helpers.rake | 35 +++++- lib/tasks/gitlab/workhorse.rake | 23 ++++ spec/support/stub_configuration.rb | 4 + spec/tasks/gitlab/task_helpers_spec.rb | 87 +++++++++++++ spec/tasks/gitlab/workhorse_rake_spec.rb | 117 ++++++++++++++++++ 9 files changed, 295 insertions(+), 47 deletions(-) create mode 100644 changelogs/unreleased/22719-provide-a-new-gitlab-workhorse-install-rake-task-similar-to-gitlab-shell-install.yml create mode 100644 lib/tasks/gitlab/workhorse.rake create mode 100644 spec/tasks/gitlab/task_helpers_spec.rb create mode 100644 spec/tasks/gitlab/workhorse_rake_spec.rb diff --git a/changelogs/unreleased/22719-provide-a-new-gitlab-workhorse-install-rake-task-similar-to-gitlab-shell-install.yml b/changelogs/unreleased/22719-provide-a-new-gitlab-workhorse-install-rake-task-similar-to-gitlab-shell-install.yml new file mode 100644 index 00000000000..54bd313f075 --- /dev/null +++ b/changelogs/unreleased/22719-provide-a-new-gitlab-workhorse-install-rake-task-similar-to-gitlab-shell-install.yml @@ -0,0 +1,4 @@ +--- +title: New `gitlab:workhorse:install` rake task +merge_request: 6574 +author: diff --git a/doc/install/installation.md b/doc/install/installation.md index ee02d6024d9..25b186b63f7 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -175,7 +175,7 @@ We recommend using a PostgreSQL database. For MySQL check the ```bash sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" ``` - + 1. Create the `pg_trgm` extension (required for GitLab 8.6+): ```bash @@ -396,15 +396,25 @@ GitLab Shell is an SSH access and repository management software developed speci ### Install gitlab-workhorse -GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). -If you are not using Linux you may have to run `gmake` instead of -`make` below. +GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The +following command-line will install GitLab-Workhorse in `home/git/gitlab-workhorse` +which is the recommended location. - cd /home/git - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git - cd gitlab-workhorse - sudo -u git -H git checkout v1.0.1 - sudo -u git -H make + cd /home/git/gitlab + + sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] RAILS_ENV=production + +You can specify a different Git repository by providing `GITLAB_WORKHORSE_REPO`: + + cd /home/git/gitlab + + sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_REPO=https://gitlab.com/gitlab-org/gitlab-ce.git RAILS_ENV=production + +You can specify a different version to use by providing `GITLAB_WORKHORSE_VERSION`: + + cd /home/git/gitlab + + sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_VERSION=0.8.1 RAILS_ENV=production ### Initialize Database and Activate Advanced Features diff --git a/doc/update/8.12-to-8.13.md b/doc/update/8.12-to-8.13.md index c0084d9d59c..8c0d3f78b55 100644 --- a/doc/update/8.12-to-8.13.md +++ b/doc/update/8.12-to-8.13.md @@ -166,7 +166,7 @@ See [smtp_settings.rb.sample] as an example. Ensure you're still up-to-date with the latest init script changes: sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab - + For Ubuntu 16.04.1 LTS: sudo systemctl daemon-reload diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 58761a129d4..5a09cd7ce41 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -5,42 +5,23 @@ namespace :gitlab do warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = 'v' + default_version - args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") + default_version_tag = "v#{default_version}" + args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') - user = Gitlab.config.gitlab.user - home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - # Clone if needed - if File.directory?(target_dir) - Dir.chdir(target_dir) do - system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet)) - system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag})) - end - else - system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) - end + checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - reseted = reset_to_commit(args) - - unless reseted - system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) - reset_to_commit(args) - end - config = { - user: user, + user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - auth_file: File.join(home_dir, ".ssh", "authorized_keys"), + auth_file: File.join(user_home, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, namespace: "resque:gitlab" @@ -74,7 +55,7 @@ namespace :gitlab do # be an issue since it is more than likely that there are no "normal" # user accounts on a gitlab server). The alternative is for the admin to # install a ruby (1.9.3+) in the global path. - File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| + File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end @@ -142,15 +123,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) - - unless status.zero? - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) - end - - tag = tag.strip - system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) - end end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index 74be413423a..85c3a3a9b0f 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -70,7 +70,7 @@ namespace :gitlab do # Runs the given command # - # Returns nil if the command was not found + # Returns '' if the command was not found # Returns the output of the command otherwise # # see also #run_and_match @@ -137,4 +137,37 @@ namespace :gitlab do def repository_storage_paths_args Gitlab.config.repositories.storages.values end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_tag(tag:, repo:, target_dir:) + if Dir.exist?(target_dir) + Dir.chdir(target_dir) do + run_command(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) + run_command(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) + end + else + run_command(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + # Make sure we're on the right tag + Dir.chdir(target_dir) do + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + reset_to_tag(tag) + end + end + + def reset_to_tag(tag_wanted) + tag, status = Gitlab::Popen.popen(%W[#{Gitlab.config.git.bin_path} describe -- #{tag_wanted}]) + + unless status.zero? + run_command(%W(#{Gitlab.config.git.bin_path} fetch origin)) + tag = run_command(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag_wanted}]) + end + + run_command(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag.strip}]) + end end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake new file mode 100644 index 00000000000..5964cb83b89 --- /dev/null +++ b/lib/tasks/gitlab/workhorse.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :workhorse do + desc "GitLab | Install or upgrade gitlab-workhorse" + task :install, [:dir] => :environment do |t, args| + warn_user_is_not_gitlab + unless args.dir.present? + abort "Please specify the directory where you want to install gitlab-workhorse:\n rake gitlab:workhorse:install[/home/git/gitlab-workhorse]" + end + + tag = "v#{ENV['GITLAB_WORKHORSE_VERSION'] || Gitlab::Workhorse.version}" + repo = ENV['GITLAB_WORKHORSE_REPO'] || 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + + _, status = Gitlab::Popen.popen(%w[which gmake]) + command = status.zero? ? 'gmake' : 'make' + + Dir.chdir(args.dir) do + run_command([command]) + end + end + end +end diff --git a/spec/support/stub_configuration.rb b/spec/support/stub_configuration.rb index f40ee862df8..c0847bbd7f1 100644 --- a/spec/support/stub_configuration.rb +++ b/spec/support/stub_configuration.rb @@ -17,6 +17,10 @@ module StubConfiguration allow(Gitlab.config.gravatar).to receive_messages(messages) end + def stub_gitlab_workhorse_setting(messages) + allow(Gitlab.config.gitlab_workhorse).to receive_messages(messages) + end + def stub_incoming_email_setting(messages) allow(Gitlab.config.incoming_email).to receive_messages(messages) end diff --git a/spec/tasks/gitlab/task_helpers_spec.rb b/spec/tasks/gitlab/task_helpers_spec.rb new file mode 100644 index 00000000000..2a2d4a39ba8 --- /dev/null +++ b/spec/tasks/gitlab/task_helpers_spec.rb @@ -0,0 +1,87 @@ +require 'spec_helper' +require 'rake' + +describe 'gitlab:workhorse namespace rake task' do + before :all do + Rake.application.rake_require 'tasks/gitlab/task_helpers' + + # empty task as env is already loaded + Rake::Task.define_task :environment + end + + describe '#checkout_or_clone_tag' do + let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-test.git' } + let(:clone_path) { Rails.root.join('tmp/tests/task_helpers_tests').to_s } + let(:tag) { 'v1.1.0' } + before do + FileUtils.rm_rf(clone_path) + allow_any_instance_of(Object).to receive(:run_command) + expect_any_instance_of(Object).to receive(:reset_to_tag).with(tag) + end + + after do + FileUtils.rm_rf(clone_path) + end + + context 'target_dir does not exist' do + it 'clones the repo, retrieve the tag from origin, and checkout the tag' do + expect(Dir).to receive(:chdir).and_call_original + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) { FileUtils.mkdir_p(clone_path) } # Fake the cloning + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) + end + end + + context 'target_dir exists' do + before do + FileUtils.mkdir_p(clone_path) + end + + it 'fetch and checkout the tag' do + expect(Dir).to receive(:chdir).twice.and_call_original + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) + end + end + end + + describe '#reset_to_tag' do + let(:tag) { 'v1.1.0' } + before do + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag}]) + end + + context 'when the tag is not checked out locally' do + before do + expect(Gitlab::Popen). + to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return(['', 42]) + end + + it 'fetch origin, ensure the tag exists, and resets --hard to the given tag' do + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch origin]) + expect_any_instance_of(Object). + to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag}]).and_return(tag) + + reset_to_tag(tag) + end + end + + context 'when the tag is checked out locally' do + before do + expect(Gitlab::Popen). + to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return([tag, 0]) + end + + it 'resets --hard to the given tag' do + reset_to_tag(tag) + end + end + end +end diff --git a/spec/tasks/gitlab/workhorse_rake_spec.rb b/spec/tasks/gitlab/workhorse_rake_spec.rb new file mode 100644 index 00000000000..c8ad004282a --- /dev/null +++ b/spec/tasks/gitlab/workhorse_rake_spec.rb @@ -0,0 +1,117 @@ +require 'spec_helper' +require 'rake' + +describe 'gitlab:workhorse namespace rake task' do + before :all do + Rake.application.rake_require 'tasks/gitlab/task_helpers' + Rake.application.rake_require 'tasks/gitlab/workhorse' + + # empty task as env is already loaded + Rake::Task.define_task :environment + end + + def run_rake_task(task_name, *args) + Rake::Task[task_name].reenable + Rake.application.invoke_task("#{task_name}[#{args.join(',')}]") + end + + describe 'install' do + let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' } + let(:clone_path) { Rails.root.join('tmp/tests/gitlab-workhorse').to_s } + let(:tag) { "v#{File.read(Rails.root.join(Gitlab::Workhorse::VERSION_FILE)).chomp}" } + before do + # avoid writing task output to spec progress + allow($stdout).to receive :write + allow(ENV).to receive(:[]) + end + + context 'no dir given' do + it 'aborts and display a help message' do + expect { run_rake_task('gitlab:workhorse:install') }.to raise_error /Please specify the directory where you want to install gitlab-workhorse/ + end + end + + context 'when an underlying Git command fail' do + it 'aborts and display a help message' do + expect_any_instance_of(Object). + to receive(:checkout_or_clone_tag).and_raise 'Git error' + + expect { run_rake_task('gitlab:workhorse:install', clone_path) }.to raise_error 'Git error' + end + end + + describe 'checkout or clone' do + before do + expect(Dir).to receive(:chdir).with(clone_path) + end + + it 'calls checkout_or_clone_tag with the right arguments' do + expect_any_instance_of(Object). + to receive(:checkout_or_clone_tag).with(tag: tag, repo: repo, target_dir: clone_path) + + run_rake_task('gitlab:workhorse:install', clone_path) + end + + context 'given a specific repo' do + before do + expect(ENV).to receive(:[]).with('GITLAB_WORKHORSE_REPO').and_return('https://gitlab.com/user1/gitlab-workhorse.git') + end + + it 'calls checkout_or_clone_tag with the given repo' do + expect_any_instance_of(Object). + to receive(:checkout_or_clone_tag).with(tag: tag, repo: 'https://gitlab.com/user1/gitlab-workhorse.git', target_dir: clone_path) + + run_rake_task('gitlab:workhorse:install', clone_path) + end + end + + context 'given a specific version' do + before do + allow(ENV).to receive(:[]).with('GITLAB_WORKHORSE_VERSION').and_return('42.42.0') + end + + it 'calls checkout_or_clone_tag with the given repo' do + expect_any_instance_of(Object). + to receive(:checkout_or_clone_tag).with(tag: 'v42.42.0', repo: repo, target_dir: clone_path) + + run_rake_task('gitlab:workhorse:install', clone_path) + end + end + end + + describe 'gmake/make' do + before do + FileUtils.mkdir_p(clone_path) + expect(Dir).to receive(:chdir).with(clone_path).and_call_original + end + + context 'gmake is available' do + before do + expect_any_instance_of(Object).to receive(:checkout_or_clone_tag) + allow_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true) + end + + it 'calls gmake in the gitlab-workhorse directory' do + expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) + expect_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true) + + run_rake_task('gitlab:workhorse:install', clone_path) + end + end + + context 'gmake is not available' do + before do + expect_any_instance_of(Object).to receive(:checkout_or_clone_tag) + allow_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true) + end + + it 'calls make in the gitlab-workhorse directory' do + expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['', 42]) + expect_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true) + + run_rake_task('gitlab:workhorse:install', clone_path) + end + end + end + end +end From a9c250eaddf758f99ac8c868dc86f4df0cc157f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 24 Nov 2016 14:19:09 +0100 Subject: [PATCH 2/3] Add #run_command! to task helpers to raise a TaskFailedError if status is not 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- doc/install/installation.md | 2 +- doc/update/8.14-to-8.15.md | 202 +++++++++++++++++++++++ lib/tasks/gitlab/task_helpers.rake | 36 +++- lib/tasks/gitlab/workhorse.rake | 2 +- spec/tasks/gitlab/task_helpers_spec.rb | 22 +-- spec/tasks/gitlab/workhorse_rake_spec.rb | 8 +- 6 files changed, 246 insertions(+), 26 deletions(-) create mode 100644 doc/update/8.14-to-8.15.md diff --git a/doc/install/installation.md b/doc/install/installation.md index 25b186b63f7..de650f0900d 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -408,7 +408,7 @@ You can specify a different Git repository by providing `GITLAB_WORKHORSE_REPO`: cd /home/git/gitlab - sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_REPO=https://gitlab.com/gitlab-org/gitlab-ce.git RAILS_ENV=production + sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_REPO=https://example.com/gitlab-workhorse.git RAILS_ENV=production You can specify a different version to use by providing `GITLAB_WORKHORSE_VERSION`: diff --git a/doc/update/8.14-to-8.15.md b/doc/update/8.14-to-8.15.md new file mode 100644 index 00000000000..9e6a3b45394 --- /dev/null +++ b/doc/update/8.14-to-8.15.md @@ -0,0 +1,202 @@ +# From 8.14 to 8.15 + +Make sure you view this update guide from the tag (version) of GitLab you would +like to install. In most cases this should be the highest numbered production +tag (without rc in it). You can select the tag in the version dropdown at the +top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + + sudo service gitlab stop + +### 2. Backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +We will continue supporting Ruby < 2.3 for the time being but we recommend you +upgrade to Ruby 2.3 if you're running a source installation, as this is the same +version that ships with our Omnibus package. + +You can check which version you are running with `ruby -v`. + +Download and compile Ruby: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.1.tar.gz +echo 'c39b4001f7acb4e334cb60a0f4df72d434bef711 ruby-2.3.1.tar.gz' | shasum --check - && tar xzf ruby-2.3.1.tar.gz +cd ruby-2.3.1 +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Get latest code + +```bash +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +``` + +For GitLab Community Edition: + +```bash +sudo -u git -H git checkout 8-15-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +sudo -u git -H git checkout 8-15-stable-ee +``` + +### 5. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v4.0.0 +``` + +### 6. Update gitlab-workhorse + +Install and compile gitlab-workhorse. This requires +[Go 1.5](https://golang.org/dl) which should already be on your system from +GitLab 8.1. + +```bash +sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] RAILS_ENV=production +``` + +### 7. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Clean up assets and cache +sudo -u git -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production +``` + +### 8. Update configuration files + +#### New configuration options for `gitlab.yml` + +There are new configuration options available for [`gitlab.yml`](config/gitlab.yml.example). View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +git diff origin/8-13-stable:config/gitlab.yml.example origin/8-15-stable:config/gitlab.yml.example +``` + +#### Git configuration + +Configure Git to generate packfile bitmaps (introduced in Git 2.0) on +the GitLab server during `git gc`. + +```sh +sudo -u git -H git config --global repack.writeBitmaps true +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +# For HTTPS configurations +git diff origin/8-13-stable:lib/support/nginx/gitlab-ssl origin/8-15-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/8-13-stable:lib/support/nginx/gitlab origin/8-15-stable:lib/support/nginx/gitlab +``` + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-15-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-15-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +Ensure you're still up-to-date with the latest init script changes: + + sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab + +For Ubuntu 16.04.1 LTS: + + sudo systemctl daemon-reload + +### 9. Start application + + sudo service gitlab start + sudo service nginx restart + +### 10. Check application status + +Check if GitLab and its environment are configured correctly: + + sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production + +To make sure you didn't miss anything run a more thorough check: + + sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (8.14) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 8.13 to 8.14](8.13-to-8.14.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index 85c3a3a9b0f..c0759b96602 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -1,4 +1,5 @@ module Gitlab + class TaskFailedError < StandardError; end class TaskAbortedByUserError < StandardError; end end @@ -81,6 +82,18 @@ namespace :gitlab do '' # if the command does not exist, return an empty string end + # Runs the given command and raise a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError unless status.zero? + + output + end + def uid_for(user_name) run_command(%W(id -u #{user_name})).chomp.to_i end @@ -145,11 +158,11 @@ namespace :gitlab do def checkout_or_clone_tag(tag:, repo:, target_dir:) if Dir.exist?(target_dir) Dir.chdir(target_dir) do - run_command(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) - run_command(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) + run_command!(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) end else - run_command(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end # Make sure we're on the right tag @@ -161,13 +174,18 @@ namespace :gitlab do end def reset_to_tag(tag_wanted) - tag, status = Gitlab::Popen.popen(%W[#{Gitlab.config.git.bin_path} describe -- #{tag_wanted}]) + tag = + begin + run_command!(%W[#{Gitlab.config.git.bin_path} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag_wanted}]) + end - unless status.zero? - run_command(%W(#{Gitlab.config.git.bin_path} fetch origin)) - tag = run_command(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag_wanted}]) + if tag + run_command!(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag.strip}]) + else + raise Gitlab::TaskFailedError end - - run_command(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag.strip}]) end end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index 5964cb83b89..563744dd0c7 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -16,7 +16,7 @@ namespace :gitlab do command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do - run_command([command]) + run_command!([command]) end end end diff --git a/spec/tasks/gitlab/task_helpers_spec.rb b/spec/tasks/gitlab/task_helpers_spec.rb index 2a2d4a39ba8..dccb3b4cf9a 100644 --- a/spec/tasks/gitlab/task_helpers_spec.rb +++ b/spec/tasks/gitlab/task_helpers_spec.rb @@ -15,7 +15,7 @@ describe 'gitlab:workhorse namespace rake task' do let(:tag) { 'v1.1.0' } before do FileUtils.rm_rf(clone_path) - allow_any_instance_of(Object).to receive(:run_command) + allow_any_instance_of(Object).to receive(:run_command!) expect_any_instance_of(Object).to receive(:reset_to_tag).with(tag) end @@ -27,7 +27,7 @@ describe 'gitlab:workhorse namespace rake task' do it 'clones the repo, retrieve the tag from origin, and checkout the tag' do expect(Dir).to receive(:chdir).and_call_original expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) { FileUtils.mkdir_p(clone_path) } # Fake the cloning + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) { FileUtils.mkdir_p(clone_path) } # Fake the cloning checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) end @@ -41,9 +41,9 @@ describe 'gitlab:workhorse namespace rake task' do it 'fetch and checkout the tag' do expect(Dir).to receive(:chdir).twice.and_call_original expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) end @@ -54,20 +54,20 @@ describe 'gitlab:workhorse namespace rake task' do let(:tag) { 'v1.1.0' } before do expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag}]) + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag}]) end context 'when the tag is not checked out locally' do before do - expect(Gitlab::Popen). - to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return(['', 42]) + expect_any_instance_of(Object). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_raise(Gitlab::TaskFailedError) end it 'fetch origin, ensure the tag exists, and resets --hard to the given tag' do expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} fetch origin]) + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} fetch origin]) expect_any_instance_of(Object). - to receive(:run_command).with(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag}]).and_return(tag) + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag}]).and_return(tag) reset_to_tag(tag) end @@ -75,8 +75,8 @@ describe 'gitlab:workhorse namespace rake task' do context 'when the tag is checked out locally' do before do - expect(Gitlab::Popen). - to receive(:popen).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return([tag, 0]) + expect_any_instance_of(Object). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return(tag) end it 'resets --hard to the given tag' do diff --git a/spec/tasks/gitlab/workhorse_rake_spec.rb b/spec/tasks/gitlab/workhorse_rake_spec.rb index c8ad004282a..87bc1b128bf 100644 --- a/spec/tasks/gitlab/workhorse_rake_spec.rb +++ b/spec/tasks/gitlab/workhorse_rake_spec.rb @@ -88,12 +88,12 @@ describe 'gitlab:workhorse namespace rake task' do context 'gmake is available' do before do expect_any_instance_of(Object).to receive(:checkout_or_clone_tag) - allow_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true) + allow_any_instance_of(Object).to receive(:run_command!).with(['gmake']).and_return(true) end it 'calls gmake in the gitlab-workhorse directory' do expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) - expect_any_instance_of(Object).to receive(:run_command).with(['gmake']).and_return(true) + expect_any_instance_of(Object).to receive(:run_command!).with(['gmake']).and_return(true) run_rake_task('gitlab:workhorse:install', clone_path) end @@ -102,12 +102,12 @@ describe 'gitlab:workhorse namespace rake task' do context 'gmake is not available' do before do expect_any_instance_of(Object).to receive(:checkout_or_clone_tag) - allow_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true) + allow_any_instance_of(Object).to receive(:run_command!).with(['make']).and_return(true) end it 'calls make in the gitlab-workhorse directory' do expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['', 42]) - expect_any_instance_of(Object).to receive(:run_command).with(['make']).and_return(true) + expect_any_instance_of(Object).to receive(:run_command!).with(['make']).and_return(true) run_rake_task('gitlab:workhorse:install', clone_path) end From b193e8497444a19e4ea541f73f82eb7e21aa8879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 29 Nov 2016 19:21:25 +0100 Subject: [PATCH 3/3] Move task helpers to a module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- doc/install/installation.md | 8 +- doc/update/8.14-to-8.15.md | 2 +- lib/tasks/gitlab/helpers.rake | 8 + lib/tasks/gitlab/task_helpers.rake | 191 ------------------ lib/tasks/gitlab/task_helpers.rb | 190 +++++++++++++++++ lib/tasks/gitlab/workhorse.rake | 2 +- spec/rake_helper.rb | 2 +- spec/support/rake_helpers.rb | 4 +- spec/support/stub_configuration.rb | 4 - spec/tasks/gitlab/backup_rake_spec.rb | 2 +- .../gitlab/mail_google_schema_whitelisting.rb | 2 +- spec/tasks/gitlab/task_helpers_spec.rb | 89 ++++---- spec/tasks/gitlab/users_rake_spec.rb | 2 +- spec/tasks/gitlab/workhorse_rake_spec.rb | 16 +- 14 files changed, 262 insertions(+), 260 deletions(-) create mode 100644 lib/tasks/gitlab/helpers.rake delete mode 100644 lib/tasks/gitlab/task_helpers.rake create mode 100644 lib/tasks/gitlab/task_helpers.rb diff --git a/doc/install/installation.md b/doc/install/installation.md index de650f0900d..77adb4c9f7b 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -397,24 +397,24 @@ GitLab Shell is an SSH access and repository management software developed speci ### Install gitlab-workhorse GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The -following command-line will install GitLab-Workhorse in `home/git/gitlab-workhorse` +following command-line will install GitLab-Workhorse in `/home/git/gitlab-workhorse` which is the recommended location. cd /home/git/gitlab - sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] RAILS_ENV=production + sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]" RAILS_ENV=production You can specify a different Git repository by providing `GITLAB_WORKHORSE_REPO`: cd /home/git/gitlab - sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_REPO=https://example.com/gitlab-workhorse.git RAILS_ENV=production + sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]" GITLAB_WORKHORSE_REPO=https://example.com/gitlab-workhorse.git RAILS_ENV=production You can specify a different version to use by providing `GITLAB_WORKHORSE_VERSION`: cd /home/git/gitlab - sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] GITLAB_WORKHORSE_VERSION=0.8.1 RAILS_ENV=production + sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]" GITLAB_WORKHORSE_VERSION=0.8.1 RAILS_ENV=production ### Initialize Database and Activate Advanced Features diff --git a/doc/update/8.14-to-8.15.md b/doc/update/8.14-to-8.15.md index 9e6a3b45394..576b943b98c 100644 --- a/doc/update/8.14-to-8.15.md +++ b/doc/update/8.14-to-8.15.md @@ -82,7 +82,7 @@ Install and compile gitlab-workhorse. This requires GitLab 8.1. ```bash -sudo -u git -H bundle exec rake gitlab:workhorse:install[/home/git/gitlab-workhorse] RAILS_ENV=production +sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]" RAILS_ENV=production ``` ### 7. Install libs, migrations, etc. diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake new file mode 100644 index 00000000000..dd2d5861481 --- /dev/null +++ b/lib/tasks/gitlab/helpers.rake @@ -0,0 +1,8 @@ +require 'tasks/gitlab/task_helpers' + +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + +namespace :gitlab do + include Gitlab::TaskHelpers +end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake deleted file mode 100644 index c0759b96602..00000000000 --- a/lib/tasks/gitlab/task_helpers.rake +++ /dev/null @@ -1,191 +0,0 @@ -module Gitlab - class TaskFailedError < StandardError; end - class TaskAbortedByUserError < StandardError; end -end - -require 'rainbow/ext/string' - -# Prevent StateMachine warnings from outputting during a cron task -StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] - -namespace :gitlab do - - # Ask if the user wants to continue - # - # Returns "yes" the user chose to continue - # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue - def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) - raise Gitlab::TaskAbortedByUserError unless answer == "yes" - end - - # Check which OS is running - # - # It will primarily use lsb_relase to determine the OS. - # It has fallbacks to Debian, SuSE, OS X and systems running systemd. - def os_name - os_name = run_command(%W(lsb_release -irs)) - os_name ||= if File.readable?('/etc/system-release') - File.read('/etc/system-release') - end - os_name ||= if File.readable?('/etc/debian_version') - debian_version = File.read('/etc/debian_version') - "Debian #{debian_version}" - end - os_name ||= if File.readable?('/etc/SuSE-release') - File.read('/etc/SuSE-release') - end - os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) - "Mac OS X #{os_x_version}" - end - os_name ||= if File.readable?('/etc/os-release') - File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] - end - os_name.try(:squish!) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while choices.present? && !choices.include?(answer) - answer - end - - # Runs the given command and matches the output against the given pattern - # - # Returns nil if nothing matched - # Returns the MatchData if the pattern matched - # - # see also #run_command - # see also String#match - def run_and_match(command, regexp) - run_command(command).try(:match, regexp) - end - - # Runs the given command - # - # Returns '' if the command was not found - # Returns the output of the command otherwise - # - # see also #run_and_match - def run_command(command) - output, _ = Gitlab::Popen.popen(command) - output - rescue Errno::ENOENT - '' # if the command does not exist, return an empty string - end - - # Runs the given command and raise a Gitlab::TaskFailedError exception if - # the command does not exit with 0 - # - # Returns the output of the command otherwise - def run_command!(command) - output, status = Gitlab::Popen.popen(command) - - raise Gitlab::TaskFailedError unless status.zero? - - output - end - - def uid_for(user_name) - run_command(%W(id -u #{user_name})).chomp.to_i - end - - def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end - end - - def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user - current_user = run_command(%W(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end - - def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end - - def user_home - Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home - end - - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - Dir.chdir(target_dir) do - run_command!(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) - end - else - run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) - end - - # Make sure we're on the right tag - Dir.chdir(target_dir) do - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - reset_to_tag(tag) - end - end - - def reset_to_tag(tag_wanted) - tag = - begin - run_command!(%W[#{Gitlab.config.git.bin_path} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end - end -end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb new file mode 100644 index 00000000000..e128738b5f8 --- /dev/null +++ b/lib/tasks/gitlab/task_helpers.rb @@ -0,0 +1,190 @@ +require 'rainbow/ext/string' + +module Gitlab + TaskFailedError = Class.new(StandardError) + TaskAbortedByUserError = Class.new(StandardError) + + module TaskHelpers + # Ask if the user wants to continue + # + # Returns "yes" the user chose to continue + # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue + def ask_to_continue + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + raise Gitlab::TaskAbortedByUserError unless answer == "yes" + end + + # Check which OS is running + # + # It will primarily use lsb_relase to determine the OS. + # It has fallbacks to Debian, SuSE, OS X and systems running systemd. + def os_name + os_name = run_command(%W(lsb_release -irs)) + os_name ||= if File.readable?('/etc/system-release') + File.read('/etc/system-release') + end + os_name ||= if File.readable?('/etc/debian_version') + debian_version = File.read('/etc/debian_version') + "Debian #{debian_version}" + end + os_name ||= if File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + end + os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + end + os_name ||= if File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + os_name.try(:squish!) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while choices.present? && !choices.include?(answer) + answer + end + + # Runs the given command and matches the output against the given pattern + # + # Returns nil if nothing matched + # Returns the MatchData if the pattern matched + # + # see also #run_command + # see also String#match + def run_and_match(command, regexp) + run_command(command).try(:match, regexp) + end + + # Runs the given command + # + # Returns '' if the command was not found + # Returns the output of the command otherwise + # + # see also #run_and_match + def run_command(command) + output, _ = Gitlab::Popen.popen(command) + output + rescue Errno::ENOENT + '' # if the command does not exist, return an empty string + end + + # Runs the given command and raises a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError unless status.zero? + + output + end + + def uid_for(user_name) + run_command(%W(id -u #{user_name})).chomp.to_i + end + + def gid_for(group_name) + begin + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" + end + end + + def warn_user_is_not_gitlab + unless @warned_user_not_gitlab + gitlab_user = Gitlab.config.gitlab.user + current_user = run_command(%W(whoami)).chomp + unless current_user == gitlab_user + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + end + @warned_user_not_gitlab = true + end + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successfull (according to their exit code) + # Returns false if any or all subcommands failed. + def auto_fix_git_config(options) + if !@warned_user_not_gitlab + command_success = options.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + else + false + end + end + + def all_repos + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_tag(tag:, repo:, target_dir:) + if Dir.exist?(target_dir) + checkout_tag(tag, target_dir) + else + clone_repo(repo, target_dir) + end + + reset_to_tag(tag, target_dir) + end + + def clone_repo(repo, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + def checkout_tag(tag, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + end + + def reset_to_tag(tag_wanted, target_dir) + tag = + begin + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) + end + + if tag + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) + else + raise Gitlab::TaskFailedError + end + end + end +end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index 563744dd0c7..46bd0bf2e7b 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -4,7 +4,7 @@ namespace :gitlab do task :install, [:dir] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? - abort "Please specify the directory where you want to install gitlab-workhorse:\n rake gitlab:workhorse:install[/home/git/gitlab-workhorse]" + abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end tag = "v#{ENV['GITLAB_WORKHORSE_VERSION'] || Gitlab::Workhorse.version}" diff --git a/spec/rake_helper.rb b/spec/rake_helper.rb index 9b5b4bf9fea..298a520f5ca 100644 --- a/spec/rake_helper.rb +++ b/spec/rake_helper.rb @@ -8,7 +8,7 @@ RSpec.configure do |config| config.before(:all) do $stdout = StringIO.new - Rake.application.rake_require 'tasks/gitlab/task_helpers' + Rake.application.rake_require 'tasks/gitlab/helpers' Rake::Task.define_task :environment end diff --git a/spec/support/rake_helpers.rb b/spec/support/rake_helpers.rb index 52d80c69835..4a8158ed79b 100644 --- a/spec/support/rake_helpers.rb +++ b/spec/support/rake_helpers.rb @@ -1,7 +1,7 @@ module RakeHelpers - def run_rake_task(task_name) + def run_rake_task(task_name, *args) Rake::Task[task_name].reenable - Rake.application.invoke_task task_name + Rake.application.invoke_task("#{task_name}[#{args.join(',')}]") end def stub_warn_user_is_not_gitlab diff --git a/spec/support/stub_configuration.rb b/spec/support/stub_configuration.rb index c0847bbd7f1..f40ee862df8 100644 --- a/spec/support/stub_configuration.rb +++ b/spec/support/stub_configuration.rb @@ -17,10 +17,6 @@ module StubConfiguration allow(Gitlab.config.gravatar).to receive_messages(messages) end - def stub_gitlab_workhorse_setting(messages) - allow(Gitlab.config.gitlab_workhorse).to receive_messages(messages) - end - def stub_incoming_email_setting(messages) allow(Gitlab.config.incoming_email).to receive_messages(messages) end diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 287d83344db..ecbfc236d3d 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -5,7 +5,7 @@ describe 'gitlab:app namespace rake task' do let(:enable_registry) { true } before :all do - Rake.application.rake_require 'tasks/gitlab/task_helpers' + Rake.application.rake_require 'tasks/gitlab/helpers' Rake.application.rake_require 'tasks/gitlab/backup' Rake.application.rake_require 'tasks/gitlab/shell' Rake.application.rake_require 'tasks/gitlab/db' diff --git a/spec/tasks/gitlab/mail_google_schema_whitelisting.rb b/spec/tasks/gitlab/mail_google_schema_whitelisting.rb index 37feb5e6faf..80fc8c48fed 100644 --- a/spec/tasks/gitlab/mail_google_schema_whitelisting.rb +++ b/spec/tasks/gitlab/mail_google_schema_whitelisting.rb @@ -3,7 +3,7 @@ require 'rake' describe 'gitlab:mail_google_schema_whitelisting rake task' do before :all do - Rake.application.rake_require "tasks/gitlab/task_helpers" + Rake.application.rake_require "tasks/gitlab/helpers" Rake.application.rake_require "tasks/gitlab/mail_google_schema_whitelisting" # empty task as env is already loaded Rake::Task.define_task :environment diff --git a/spec/tasks/gitlab/task_helpers_spec.rb b/spec/tasks/gitlab/task_helpers_spec.rb index dccb3b4cf9a..86e42d845ce 100644 --- a/spec/tasks/gitlab/task_helpers_spec.rb +++ b/spec/tasks/gitlab/task_helpers_spec.rb @@ -1,86 +1,95 @@ require 'spec_helper' -require 'rake' +require 'tasks/gitlab/task_helpers' -describe 'gitlab:workhorse namespace rake task' do - before :all do - Rake.application.rake_require 'tasks/gitlab/task_helpers' +class TestHelpersTest + include Gitlab::TaskHelpers +end - # empty task as env is already loaded - Rake::Task.define_task :environment - end +describe Gitlab::TaskHelpers do + subject { TestHelpersTest.new } + + let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-test.git' } + let(:clone_path) { Rails.root.join('tmp/tests/task_helpers_tests').to_s } + let(:tag) { 'v1.1.0' } describe '#checkout_or_clone_tag' do - let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-test.git' } - let(:clone_path) { Rails.root.join('tmp/tests/task_helpers_tests').to_s } - let(:tag) { 'v1.1.0' } before do - FileUtils.rm_rf(clone_path) - allow_any_instance_of(Object).to receive(:run_command!) - expect_any_instance_of(Object).to receive(:reset_to_tag).with(tag) - end - - after do - FileUtils.rm_rf(clone_path) + allow(subject).to receive(:run_command!) + expect(subject).to receive(:reset_to_tag).with(tag, clone_path) end context 'target_dir does not exist' do it 'clones the repo, retrieve the tag from origin, and checkout the tag' do - expect(Dir).to receive(:chdir).and_call_original - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) { FileUtils.mkdir_p(clone_path) } # Fake the cloning + expect(subject).to receive(:clone_repo).with(repo, clone_path) - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) + subject.checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) end end context 'target_dir exists' do before do - FileUtils.mkdir_p(clone_path) + expect(Dir).to receive(:exist?).and_return(true) end it 'fetch and checkout the tag' do - expect(Dir).to receive(:chdir).twice.and_call_original - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} fetch --tags --quiet]) - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} checkout --quiet #{tag}]) + expect(subject).to receive(:checkout_tag).with(tag, clone_path) - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) + subject.checkout_or_clone_tag(tag: tag, repo: repo, target_dir: clone_path) end end end + describe '#clone_repo' do + it 'clones the repo in the target dir' do + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{clone_path}]) + + subject.clone_repo(repo, clone_path) + end + end + + describe '#checkout_tag' do + it 'clones the repo in the target dir' do + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} fetch --tags --quiet]) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} checkout --quiet #{tag}]) + + subject.checkout_tag(tag, clone_path) + end + end + describe '#reset_to_tag' do let(:tag) { 'v1.1.0' } before do - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} reset --hard #{tag}]) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} reset --hard #{tag}]) end context 'when the tag is not checked out locally' do before do - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_raise(Gitlab::TaskFailedError) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} describe -- #{tag}]).and_raise(Gitlab::TaskFailedError) end it 'fetch origin, ensure the tag exists, and resets --hard to the given tag' do - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} fetch origin]) - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- origin/#{tag}]).and_return(tag) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} fetch origin]) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} describe -- origin/#{tag}]).and_return(tag) - reset_to_tag(tag) + subject.reset_to_tag(tag, clone_path) end end context 'when the tag is checked out locally' do before do - expect_any_instance_of(Object). - to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} describe -- #{tag}]).and_return(tag) + expect(subject). + to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} -C #{clone_path} describe -- #{tag}]).and_return(tag) end it 'resets --hard to the given tag' do - reset_to_tag(tag) + subject.reset_to_tag(tag, clone_path) end end end diff --git a/spec/tasks/gitlab/users_rake_spec.rb b/spec/tasks/gitlab/users_rake_spec.rb index e6ebef82b78..972670e7f91 100644 --- a/spec/tasks/gitlab/users_rake_spec.rb +++ b/spec/tasks/gitlab/users_rake_spec.rb @@ -5,7 +5,7 @@ describe 'gitlab:users namespace rake task' do let(:enable_registry) { true } before :all do - Rake.application.rake_require 'tasks/gitlab/task_helpers' + Rake.application.rake_require 'tasks/gitlab/helpers' Rake.application.rake_require 'tasks/gitlab/users' # empty task as env is already loaded diff --git a/spec/tasks/gitlab/workhorse_rake_spec.rb b/spec/tasks/gitlab/workhorse_rake_spec.rb index 87bc1b128bf..b695abce091 100644 --- a/spec/tasks/gitlab/workhorse_rake_spec.rb +++ b/spec/tasks/gitlab/workhorse_rake_spec.rb @@ -1,18 +1,8 @@ -require 'spec_helper' -require 'rake' +require 'rake_helper' describe 'gitlab:workhorse namespace rake task' do before :all do - Rake.application.rake_require 'tasks/gitlab/task_helpers' Rake.application.rake_require 'tasks/gitlab/workhorse' - - # empty task as env is already loaded - Rake::Task.define_task :environment - end - - def run_rake_task(task_name, *args) - Rake::Task[task_name].reenable - Rake.application.invoke_task("#{task_name}[#{args.join(',')}]") end describe 'install' do @@ -20,13 +10,13 @@ describe 'gitlab:workhorse namespace rake task' do let(:clone_path) { Rails.root.join('tmp/tests/gitlab-workhorse').to_s } let(:tag) { "v#{File.read(Rails.root.join(Gitlab::Workhorse::VERSION_FILE)).chomp}" } before do - # avoid writing task output to spec progress - allow($stdout).to receive :write allow(ENV).to receive(:[]) end context 'no dir given' do it 'aborts and display a help message' do + # avoid writing task output to spec progress + allow($stderr).to receive :write expect { run_rake_task('gitlab:workhorse:install') }.to raise_error /Please specify the directory where you want to install gitlab-workhorse/ end end