diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb
index f85d435f58..db2fefa65e 100644
--- a/lib/rubygems/commands/setup_command.rb
+++ b/lib/rubygems/commands/setup_command.rb
@@ -361,17 +361,17 @@ By default, this RubyGems will install gem as:
 
   def install_default_bundler_gem(bin_dir)
     current_default_spec = Gem::Specification.default_stubs.find {|s| s.name == "bundler" }
-    specs_dir = if current_default_spec
+    specs_dir = if current_default_spec && default_dir == Gem.default_dir
       Gem::Specification.remove_spec current_default_spec
       loaded_from = current_default_spec.loaded_from
       File.delete(loaded_from)
       File.dirname(loaded_from)
     else
-      File.join(default_dir, "specifications", "default")
+      target_specs_dir = File.join(default_dir, "specifications", "default")
+      mkdir_p target_specs_dir, :mode => 0755
+      target_specs_dir
     end
 
-    mkdir_p specs_dir, :mode => 0755
-
     bundler_spec = Dir.chdir("bundler") { Gem::Specification.load("bundler.gemspec") }
     default_spec_path = File.join(specs_dir, "#{bundler_spec.full_name}.gemspec")
     Gem.write_binary(default_spec_path, bundler_spec.to_ruby)
diff --git a/test/rubygems/test_gem_commands_setup_command.rb b/test/rubygems/test_gem_commands_setup_command.rb
index 161c88d683..aa5feb9074 100644
--- a/test/rubygems/test_gem_commands_setup_command.rb
+++ b/test/rubygems/test_gem_commands_setup_command.rb
@@ -279,6 +279,14 @@ class TestGemCommandsSetupCommand < Gem::TestCase
 
     @cmd.install_default_bundler_gem bin_dir
 
+    # leaves other versions of bundler gemspecs on default specification directory.
+    assert_path_exist previous_bundler_specification_path
+    assert_path_not_exist new_bundler_specification_path
+
+    # installs the updated bundler gemspec to destdir
+    assert_path_not_exist prepend_destdir(destdir, previous_bundler_specification_path)
+    assert_path_exist prepend_destdir(destdir, new_bundler_specification_path)
+
     bundler_spec.executables.each do |e|
       assert_path_exist prepend_destdir(destdir, File.join(@gemhome, 'gems', bundler_spec.full_name, bundler_spec.bindir, e))
     end