diff --git a/spec/bundler/quality_spec.rb b/spec/bundler/quality_spec.rb
index 31b71269b1..a266c2008d 100644
--- a/spec/bundler/quality_spec.rb
+++ b/spec/bundler/quality_spec.rb
@@ -216,7 +216,7 @@ RSpec.describe "The library itself" do
   end
 
   it "ships the correct set of files" do
-    git_list = shipped_files
+    git_list = git_ls_files(ruby_core? ? "lib/bundler lib/bundler.rb man/bundle* man/gemfile* libexec/bundle*" : "lib man exe CHANGELOG.md LICENSE.md README.md bundler.gemspec")
 
     gem_list = loaded_gemspec.files
 
diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb
index 731a2d9664..e487a86045 100644
--- a/spec/bundler/support/helpers.rb
+++ b/spec/bundler/support/helpers.rb
@@ -331,7 +331,7 @@ module Spec
 
         build_metadata = {
           :built_at => loaded_gemspec.date.utc.strftime("%Y-%m-%d"),
-          :git_commit_sha => sys_exec("git rev-parse --short HEAD", :dir => source_root).strip,
+          :git_commit_sha => git_commit_sha,
         }
 
         replace_build_metadata(build_metadata, dir: build_path) # rubocop:disable Style/HashSyntax
diff --git a/spec/bundler/support/path.rb b/spec/bundler/support/path.rb
index 187ccefd71..f56fdfc5aa 100644
--- a/spec/bundler/support/path.rb
+++ b/spec/bundler/support/path.rb
@@ -68,7 +68,7 @@ module Spec
     end
 
     def shipped_files
-      @shipped_files ||= git_ls_files(shipped_files_glob)
+      @shipped_files ||= loaded_gemspec.files
     end
 
     def lib_tracked_files
@@ -231,20 +231,22 @@ module Spec
       end
     end
 
+    def git_commit_sha
+      ruby_core_tarball? ? "unknown" : sys_exec("git rev-parse --short HEAD", :dir => source_root).strip
+    end
+
   private
 
     def git_ls_files(glob)
-      sys_exec("git ls-files -z -- #{glob}", :dir => source_root).split("\x0")
+      skip "Not running on a git context, since running tests from a tarball" if ruby_core_tarball?
+
+      sys_exec!("git ls-files -z -- #{glob}", :dir => source_root).split("\x0")
     end
 
     def tracked_files_glob
       ruby_core? ? "lib/bundler lib/bundler.rb spec/bundler man/bundle*" : ""
     end
 
-    def shipped_files_glob
-      ruby_core? ? "lib/bundler lib/bundler.rb man/bundle* man/gemfile* libexec/bundle*" : "lib man exe CHANGELOG.md LICENSE.md README.md bundler.gemspec"
-    end
-
     def lib_tracked_files_glob
       ruby_core? ? "lib/bundler lib/bundler.rb" : "lib"
     end
@@ -253,6 +255,14 @@ module Spec
       ruby_core? ? "man/bundle* man/gemfile*" : "man"
     end
 
+    def git_root
+      ruby_core? ? source_root : source_root.parent
+    end
+
+    def ruby_core_tarball?
+      !git_root.join(".git").directory?
+    end
+
     extend self
   end
 end