Merge branch 'gitaly-install-rake' into 'master'

Docs and scripts to install Gitaly from source

Closes gitaly#136 and #28446

See merge request !9941

bin/with_env

@@ -0,0 +1,16 @@
+#!/bin/sh
+# Usage: with_env ENV_FILE COMMAND [ARGS...]
+#
+# This script lets you modify the environment of an executable before
+# launching it. It uses an 'env file' which must contain lines like
+# 'MY_VARIABLE="my value"'.
+#
+env_file=$1
+shift
+
+# Use set -a to export all variables defined in env_file.
+set -a
+. "${env_file}"
+set +a
+
+exec "$@"

config/gitlab.yml.example

@@ -450,7 +450,7 @@ production: &base
     # This setting is obsolete because we expect it to be moved under
     # repositories/storages in GitLab 9.1.
     #
-    # socket_path: tmp/sockets/gitaly.socket
+    # socket_path: tmp/sockets/private/gitaly.socket
 
   #
   # 4. Advanced settings

doc/administration/gitaly/index.md

@@ -0,0 +1,86 @@
+# Gitaly
+
+[Gitaly](https://gitlab.com/gitlab-org/gitlay) (introduced in GitLab
+9.0) is a service that provides high-level RPC access to Git
+repositories. As of GitLab 9.0 it is still an optional component with
+limited scope.
+
+GitLab components that access Git repositories (gitlab-rails,
+gitlab-shell, gitlab-workhorse) act as clients to Gitaly. End users do
+not have direct access to Gitaly.
+
+## Configuring Gitaly
+
+The Gitaly service itself is configured via environment variables.
+These variables are documented [in the gitaly
+repository](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md).
+
+To change a Gitaly environment variable in Omnibus you can use
+`gitaly['env']` in `/etc/gitlab/gitlab.rb`. Changes will be applied
+when you run `gitlab-ctl reconfigure`.
+
+```ruby
+gitaly['env'] = {
+  'GITALY_MY_VARIABLE' => 'value'
+}
+```
+
+To change a Gitaly environment variable in installations from source
+you can edit `/home/git/gitaly/env`.
+
+```shell
+GITALY_MY_VARIABLE='value'
+```
+
+Changes to `/home/git/gitaly/env` are applied when you run `service
+gitlab restart`.
+
+## Configuring GitLab to not use Gitaly
+
+Gitaly is still an optional component in GitLab 9.0. This means you
+can choose to not use it.
+
+In Omnibus you can make the following change in
+`/etc/gitlab/gitlab.rb` and reconfigure. This will both disable the
+Gitaly service and configure the rest of GitLab not to use it.
+
+```ruby
+gitaly['enable'] = false
+```
+
+In source installations, edit `/home/git/gitlab/config/gitlab.yml` and
+make sure `socket_path` in the `gitaly` section is commented out. This
+does not disable the Gitaly service; it only prevents it from being
+used.
+
+Apply the change with `service gitlab restart`.
+
+```yaml
+  gitaly:
+    # socket_path: tmp/sockets/private/gitlay.socket
+```
+
+## Disabling or enabling the Gitaly service
+
+Be careful: if you disable Gitaly without instructing the rest of your
+GitLab installation not to use Gitaly, you may end up with errors
+because GitLab tries to access a service that is not running.
+
+To disable the Gitaly service in your Omnibus installation, add the
+following line to `/etc/gitlab/gitlab.rb`:
+
+```ruby
+gitaly['enable'] = false
+```
+
+When you run `gitlab-ctl reconfigure` the Gitaly service will be
+disabled.
+
+To disable the Gitaly service in an installation from source, add the
+following to `/etc/default/gitlab`:
+
+```shell
+gitaly_enabled=false
+```
+
+When you run `service gitlab restart` Gitaly will be disabled.

doc/install/installation.md

@@ -456,6 +456,36 @@ Make GitLab start on boot:
 
     sudo update-rc.d gitlab defaults 21
 
+### Install Gitaly
+
+As of GitLab 9.0 Gitaly is an **optional** component. Its
+configuration is expected to change in GitLab 9.1. It is OK to wait
+with setting up Gitaly until you upgrade to GitLab 9.1 or later.
+
+    # Fetch Gitaly source with Git and compile with Go
+    sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production
+
+    # Restrict Gitaly socket access
+    sudo chmod 0700 /home/git/gitlab/tmp/sockets/private
+    sudo chown git /home/git/gitlab/tmp/sockets/private
+
+    # Configure Gitaly
+    echo 'GITALY_SOCKET_PATH=/home/git/gitlab/tmp/sockets/private/gitaly.socket' | \
+      sudo -u git tee -a /home/git/gitaly/env
+
+    # Enable Gitaly in the init script
+    echo 'gitaly_enabled=true' | sudo tee -a /etc/default/gitlab
+
+Next, edit `/home/git/gitlab/config/gitlab.yml` and make sure `socket_path` in
+the `gitaly:` section is uncommented.
+
+    # <- gitlab.yml indentation starts here
+      gitaly:
+        socket_path: tmp/sockets/private/gitaly.socket
+
+For more information about configuring Gitaly see
+[doc/administration/gitaly](../administration/gitaly).
+
 ### Setup Logrotate
 
     sudo cp lib/support/logrotate/gitlab /etc/logrotate.d/gitlab

doc/update/8.17-to-9.0.md

@@ -145,7 +145,15 @@ sudo -u git -H git fetch --all --tags
 sudo -u git -H git checkout v5.0.0
 ```
 
-### 9. Update configuration files
+### 9. Optional: install Gitaly
+
+Gitaly is still an optional component of GitLab. If you want to save time
+during your 9.0 upgrade **you can skip this step**.
+
+If you do want to set up Gitaly in GitLab 9.0 then follow [Gitaly section of the installation
+guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/9-0-stable/doc/install/installation.md#install-gitaly).
+
+### 10. Update configuration files
 
 #### New configuration options for `gitlab.yml`
 
@@ -282,14 +290,14 @@ For Ubuntu 16.04.1 LTS:
 sudo systemctl daemon-reload
 ```
 
-### 10. Start application
+### 11. Start application
 
 ```bash
 sudo service gitlab start
 sudo service nginx restart
 ```
 
-### 11. Check application status
+### 12. Check application status
 
 Check if GitLab and its environment are configured correctly:
 

lib/gitlab/gitaly_client.rb

@@ -2,6 +2,8 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+
     def self.gitaly_address
       if Gitlab.config.gitaly.socket_path
         "unix://#{Gitlab.config.gitaly.socket_path}"
@@ -39,5 +41,10 @@ module Gitlab
         yield is_enabled
       end
     end
+
+    def self.expected_server_version
+      path = Rails.root.join(SERVER_VERSION_FILE)
+      path.read.chomp
+    end
   end
 end

lib/support/init.d/gitlab

@@ -48,6 +48,10 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
 gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
+gitaly_enabled=false
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
+gitaly_log="$app_root/log/gitaly.log"
 
 # Read configuration variable file if it is present
 test -f /etc/default/gitlab && . /etc/default/gitlab
@@ -101,13 +105,20 @@ check_pids(){
       gppid=0
     fi
   fi
+  if [ "$gitaly_enabled" = true ]; then
+    if [ -f "$gitaly_pid_path" ]; then
+      gapid=$(cat "$gitaly_pid_path")
+    else
+      gapid=0
+    fi
+  fi
 }
 
 ## Called when we have started the two processes and are waiting for their pid files.
 wait_for_pids(){
   # We are sleeping a bit here mostly because sidekiq is slow at writing its pid
   i=0;
-  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; }; do
+  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; } || { [ "$gitaly_enabled" = true ] && [ ! -f $gitaly_pid_path ]; }; do
     sleep 0.1;
     i=$((i+1))
     if [ $((i%10)) = 0 ]; then
@@ -164,7 +175,15 @@ check_status(){
       gitlab_pages_status="-1"
     fi
   fi
-  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; }; then
+  if [ "$gitaly_enabled" = true ]; then
+    if [ $gapid -ne 0 ]; then
+      kill -0 "$gapid" 2>/dev/null
+      gitaly_status="$?"
+    else
+      gitaly_status="-1"
+    fi
+  fi
+  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; } && { [ "$gitaly_enabled" != true ] || [ $gitaly_status = 0 ]; }; then
     gitlab_status=0
   else
     # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
@@ -213,12 +232,19 @@ check_stale_pids(){
       exit 1
     fi
   fi
+  if [ "$gitaly_enabled" = true ] && [ "$gapid" != "0" ] && [ "$gitaly_status" != "0" ]; then
+    echo "Removing stale Gitaly pid. This is most likely caused by Gitaly crashing the last time it ran."
+    if ! rm "$gitaly_pid_path"; then
+      echo "Unable to remove stale pid, exiting"
+      exit 1
+    fi
+  fi
 }
 
 ## If no parts of the service is running, bail out.
 exit_if_not_running(){
   check_stale_pids
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     exit
   fi
@@ -243,6 +269,9 @@ start_gitlab() {
   if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then
     echo "Starting GitLab Pages"
   fi
+  if [ "$gitaly_enabled" = true ] && [ "$gitaly_status" != "0" ]; then
+    echo "Starting Gitaly"
+  fi
 
   # Then check if the service is running. If it is: don't start again.
   if [ "$web_status" = "0" ]; then
@@ -292,6 +321,16 @@ start_gitlab() {
     fi
   fi
 
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+      echo "Gitaly is already running with pid $gapid, not restarting"
+    else
+      $app_root/bin/daemon_with_pidfile $gitaly_pid_path \
+        $app_root/bin/with_env $gitaly_dir/env \
+          $gitaly_dir/gitaly >> $gitaly_log 2>&1 &
+    fi
+  fi
+
   # Wait for the pids to be planted
   wait_for_pids
   # Finally check the status to tell wether or not GitLab is running
@@ -322,13 +361,17 @@ stop_gitlab() {
     echo "Shutting down gitlab-pages"
     kill -- $(cat $gitlab_pages_pid_path)
   fi
+  if [ "$gitaly_status" = "0" ]; then
+    echo "Shutting down Gitaly"
+    kill -- $(cat $gitaly_pid_path)
+  fi
 
   # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
-  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; do
+  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; do
     sleep 1
     check_status
     printf "."
-    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
       printf "\n"
       break
     fi
@@ -343,6 +386,7 @@ stop_gitlab() {
     rm "$mail_room_pid_path" 2>/dev/null
   fi
   rm -f "$gitlab_pages_pid_path"
+  rm -f "$gitaly_pid_path"
 
   print_status
 }
@@ -350,7 +394,7 @@ stop_gitlab() {
 ## Prints the status of GitLab and its components.
 print_status() {
   check_status
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     return
   fi
@@ -383,7 +427,14 @@ print_status() {
         printf "The GitLab Pages is \033[31mnot running\033[0m.\n"
     fi
   fi
-  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; }; then
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+        echo "Gitaly with pid $gapid is running."
+    else
+        printf "Gitaly is \033[31mnot running\033[0m.\n"
+    fi
+  fi
+  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then
     printf "GitLab and all its components are \033[32mup and running\033[0m.\n"
   fi
 }
@@ -414,7 +465,7 @@ reload_gitlab(){
 ## Restarts Sidekiq and Unicorn.
 restart_gitlab(){
   check_status
-  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; then
+  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then
     stop_gitlab
   fi
   start_gitlab

lib/support/init.d/gitlab.default.example

@@ -84,3 +84,7 @@ mail_room_pid_path="$pid_path/mail_room.pid"
 # shell other than "bash"
 # The default is "/bin/bash"
 shell_path="/bin/bash"
+
+# This variable controls whether the init script starts/stops Gitaly
+gitaly_enabled=false
+gitaly_log="$app_root/log/gitaly.log"

lib/tasks/gitlab/gitaly.rake

@@ -0,0 +1,23 @@
+namespace :gitlab do
+  namespace :gitaly do
+    desc "GitLab | Install or upgrade gitaly"
+    task :install, [:dir] => :environment do |t, args|
+      warn_user_is_not_gitlab
+      unless args.dir.present?
+        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      end
+
+      tag = "v#{Gitlab::GitalyClient.expected_server_version}"
+      repo = 'https://gitlab.com/gitlab-org/gitaly.git'
+
+      checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+
+      _, status = Gitlab::Popen.popen(%w[which gmake])
+      command = status.zero? ? 'gmake' : 'make'
+
+      Dir.chdir(args.dir) do
+        run_command!([command])
+      end
+    end
+  end
+end

lib/tasks/gitlab/task_helpers.rb

@@ -81,7 +81,7 @@ module Gitlab
     def run_command!(command)
       output, status = Gitlab::Popen.popen(command)
 
-      raise Gitlab::TaskFailedError unless status.zero?
+      raise Gitlab::TaskFailedError.new(output) unless status.zero?
 
       output
     end

spec/tasks/gitlab/gitaly_rake_spec.rb

@@ -0,0 +1,78 @@
+require 'rake_helper'
+
+describe 'gitlab:gitaly namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/gitaly'
+  end
+
+  describe 'install' do
+    let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
+    let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
+    let(:tag) { "v#{File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp}" }
+
+    context 'no dir given' do
+      it 'aborts and display a help message' do
+        # avoid writing task output to spec progress
+        allow($stderr).to receive :write
+        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/
+      end
+    end
+
+    context 'when an underlying Git command fail' do
+      it 'aborts and display a help message' do
+        expect_any_instance_of(Object).
+          to receive(:checkout_or_clone_tag).and_raise 'Git error'
+
+        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error'
+      end
+    end
+
+    describe 'checkout or clone' do
+      before do
+        expect(Dir).to receive(:chdir).with(clone_path)
+      end
+
+      it 'calls checkout_or_clone_tag with the right arguments' do
+        expect_any_instance_of(Object).
+          to receive(:checkout_or_clone_tag).with(tag: tag, repo: repo, target_dir: clone_path)
+
+        run_rake_task('gitlab:gitaly:install', clone_path)
+      end
+    end
+
+    describe 'gmake/make' do
+      before do
+        FileUtils.mkdir_p(clone_path)
+        expect(Dir).to receive(:chdir).with(clone_path).and_call_original
+      end
+
+      context 'gmake is available' do
+        before do
+          expect_any_instance_of(Object).to receive(:checkout_or_clone_tag)
+          allow_any_instance_of(Object).to receive(:run_command!).with(['gmake']).and_return(true)
+        end
+
+        it 'calls gmake in the gitaly directory' do
+          expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
+          expect_any_instance_of(Object).to receive(:run_command!).with(['gmake']).and_return(true)
+
+          run_rake_task('gitlab:gitaly:install', clone_path)
+        end
+      end
+
+      context 'gmake is not available' do
+        before do
+          expect_any_instance_of(Object).to receive(:checkout_or_clone_tag)
+          allow_any_instance_of(Object).to receive(:run_command!).with(['make']).and_return(true)
+        end
+
+        it 'calls make in the gitaly directory' do
+          expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['', 42])
+          expect_any_instance_of(Object).to receive(:run_command!).with(['make']).and_return(true)
+
+          run_rake_task('gitlab:gitaly:install', clone_path)
+        end
+      end
+    end
+  end
+end

spec/tasks/gitlab/workhorse_rake_spec.rb

@@ -9,9 +9,6 @@ describe 'gitlab:workhorse namespace rake task' do
     let(:repo) { 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' }
     let(:clone_path) { Rails.root.join('tmp/tests/gitlab-workhorse').to_s }
     let(:tag) { "v#{File.read(Rails.root.join(Gitlab::Workhorse::VERSION_FILE)).chomp}" }
-    before do
-      allow(ENV).to receive(:[])
-    end
 
     context 'no dir given' do
       it 'aborts and display a help message' do