diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f9c5ebe7a35..bb1c84ab918 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,7 +6,7 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git
     - gitlab-org
 
 .default-cache: &default-cache
-  key: "ruby-235-with-yarn"
+  key: "ruby-2.3.6-with-yarn"
   paths:
     - vendor/ruby
     - .yarn-cache/
diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build
index 95d9fe0f176..b42ae2a2595 100755
--- a/scripts/gitaly-test-build
+++ b/scripts/gitaly-test-build
@@ -9,11 +9,21 @@ require 'fileutils'
 # called 'bundle install' using a different Gemfile, as happens with
 # gitlab-ce and gitaly.
 
-dir = 'tmp/tests/gitaly'
+tmp_tests_gitaly_dir = File.expand_path('../tmp/tests/gitaly', __dir__)
 
-abort 'gitaly build failed' unless system('make', chdir: dir)
+# Use the top-level bundle vendor folder so that we don't reinstall gems twice
+bundle_vendor_path = File.expand_path('../vendor', __dir__)
+
+env = {
+  # This ensure the `clean` config set in `scripts/prepare_build.sh` isn't taken into account
+  'BUNDLE_IGNORE_CONFIG' => 'true',
+  'BUNDLE_GEMFILE' => File.join(tmp_tests_gitaly_dir, 'ruby', 'Gemfile'),
+  'BUNDLE_FLAGS' => "--jobs=4 --path=#{bundle_vendor_path} --retry=3"
+}
+
+abort 'gitaly build failed' unless system(env, 'make', chdir: tmp_tests_gitaly_dir)
 
 # Make the 'gitaly' executable look newer than 'GITALY_SERVER_VERSION'.
 # Without this a gitaly executable created in the setup-test-env job
 # will look stale compared to GITALY_SERVER_VERSION.
-FileUtils.touch(File.join(dir, 'gitaly'), mtime: Time.now + (1 << 24))
+FileUtils.touch(File.join(tmp_tests_gitaly_dir, 'gitaly'), mtime: Time.now + (1 << 24))
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index ea406aadf39..206d62dbc78 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -3,7 +3,7 @@
 export SETUP_DB=${SETUP_DB:-true}
 export CREATE_DB_USER=${CREATE_DB_USER:-$SETUP_DB}
 export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
-export BUNDLE_INSTALL_FLAGS="--without production --jobs $(nproc) --path vendor --retry 3 --quiet"
+export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet"
 
 if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
     bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check