mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
603edfcaa0
The main thread may detect that installation has finished and thus abort, while the thread responsible for installing the spec that failed has not yet set the error message. In this case, the install process will abort with a misterious "empty" exception. I can be force the issue to reproduce by applying the following patch: ```diff diff --git a/bundler/lib/bundler/installer/parallel_installer.rb b/bundler/lib/bundler/installer/parallel_installer.rb index 3dee9f4664..7827a11d11 100644 --- a/bundler/lib/bundler/installer/parallel_installer.rb +++ b/bundler/lib/bundler/installer/parallel_installer.rb @@ -166,6 +166,7 @@ module Bundler spec_install.post_install_message = message unless message.nil? else spec_install.state = :failed + sleep 1 spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}" end Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install) @@ -183,6 +184,7 @@ module Bundler end def finished_installing? + sleep 0.5 @specs.all? do |spec| return true if spec.failed? spec.installed? diff --git a/bundler/lib/bundler/rubygems_gem_installer.rb b/bundler/lib/bundler/rubygems_gem_installer.rb index 8ce33c3953..c585cd517b 100644 --- a/bundler/lib/bundler/rubygems_gem_installer.rb +++ b/bundler/lib/bundler/rubygems_gem_installer.rb @@ -42,6 +42,7 @@ module Bundler return true unless source = @package.instance_variable_get(:@gem) return true unless source.respond_to?(:with_read_io) digest = source.with_read_io do |io| + raise BundlerError, "asdafss" digest = SharedHelpers.digest(:SHA256).new digest << io.read(16_384) until io.eof? io.rewind ``` and running `bin/rspec spec/install/gems/compact_index_spec.rb:892` will result in ``` Run options: include {:locations=>{"./spec/install/gems/compact_index_spec.rb"=>[892]}} exclude {:jruby=>true, :readline=>false, :permissions=>false, :no_color_tty=>false, :ruby_repo=>false, :bundler=>"!= 2", :git=>"!= 2.26.2", :rubygems=>"!= 3.2.0.pre1", :realworld=>true, :sudo=>true} Randomized with seed 59277 F Retried examples: 0 Failures: 1) compact index api checksum validation raises when the checksum is the wrong length Failure/Error: expect(err).to include("The given checksum for rack-1.0.0 (\"checksum!\") is not a valid SHA256 hexdigest nor base64digest") expected "" to include "The given checksum for rack-1.0.0 (\"checksum!\") is not a valid SHA256 hexdigest nor base64digest" Commands: $ /home/deivid/.rbenv/versions/2.7.1/bin/ruby -I/home/deivid/Code/rubygems/rubygems/bundler/spec -r/home/deivid/Code/rubygems/rubygems/bundler/spec/support/artifice/compact_index_wrong_gem_checksum.rb -r/home/deivid/Code/rubygems/rubygems/bundler/spec/support/hax.rb /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle install --verbose --retry 0 Running `bundle install --retry 0 --verbose` with bundler 2.2.0.dev Found changes from the lockfile, re-resolving dependencies because the list of sources changed, the dependencies in your gemfile changed, you added a new platform to your gemfile HTTP GET http://localgemserver.test/versions HTTP 200 OK http://localgemserver.test/versions Fetching gem metadata from http://localgemserver.test/ Looking up gems ["rack"] HTTP GET http://localgemserver.test/info/rack HTTP 200 OK http://localgemserver.test/info/rack Resolving dependencies... Using bundler 2.2.0.dev 0: bundler (2.2.0.dev) from /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/specifications/bundler-2.2.0.dev.gemspec Fetching rack 1.0.0 Installing rack 1.0.0 Bundler::InstallError: /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:199:in `handle_error' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:102:in `call' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:78:in `call' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:271:in `install_in_parallel' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:197:in `install' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:92:in `block in run' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:12:in `block in lock' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:9:in `open' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:9:in `lock' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:73:in `run' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:25:in `install' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli/install.rb:66:in `run' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:261:in `block in install' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/settings.rb:121:in `temporary' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:260:in `install' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/invocation.rb:127:in `invoke_command' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor.rb:399:in `dispatch' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:30:in `dispatch' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/base.rb:476:in `start' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:24:in `start' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/exe/bundle:49:in `block in <top (required)>' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/friendly_errors.rb:117:in `with_friendly_errors' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/exe/bundle:37:in `<top (required)>' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle:23:in `load' /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle:23:in `<main>' # $? => 5 # ./spec/install/gems/compact_index_spec.rb:892:in `block (3 levels) in <top (required)>' # ./spec/spec_helper.rb:104:in `block (4 levels) in <top (required)>' # ./spec/spec_helper.rb:104:in `block (3 levels) in <top (required)>' # ./spec/support/helpers.rb:352:in `block in with_gem_path_as' # ./spec/support/helpers.rb:366:in `without_env_side_effects' # ./spec/support/helpers.rb:348:in `with_gem_path_as' # ./spec/spec_helper.rb:101:in `block (2 levels) in <top (required)>' # ./spec/spec_helper.rb:73:in `block (2 levels) in <top (required)>' # ./spec/support/rubygems_ext.rb:90:in `load' # ./spec/support/rubygems_ext.rb:90:in `gem_load_and_activate' # ./spec/support/rubygems_ext.rb:18:in `gem_load' Finished in 3.01 seconds (files took 0.14209 seconds to load) 1 example, 1 failure Failed examples: rspec ./spec/install/gems/compact_index_spec.rb:886 # compact index api checksum validation raises when the checksum is the wrong length Randomized with seed 59277 ``` Without any mention to `BundlerError` and the original "asdafss" message. Fix the issue by making sure the error message is set before the ":failed" status. https://github.com/rubygems/rubygems/commit/83c8feb2c4
229 lines
6.4 KiB
Ruby
229 lines
6.4 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require_relative "../worker"
|
|
require_relative "gem_installer"
|
|
|
|
module Bundler
|
|
class ParallelInstaller
|
|
class SpecInstallation
|
|
attr_accessor :spec, :name, :post_install_message, :state, :error
|
|
def initialize(spec)
|
|
@spec = spec
|
|
@name = spec.name
|
|
@state = :none
|
|
@post_install_message = ""
|
|
@error = nil
|
|
end
|
|
|
|
def installed?
|
|
state == :installed
|
|
end
|
|
|
|
def enqueued?
|
|
state == :enqueued
|
|
end
|
|
|
|
def failed?
|
|
state == :failed
|
|
end
|
|
|
|
def installation_attempted?
|
|
installed? || failed?
|
|
end
|
|
|
|
# Only true when spec in neither installed nor already enqueued
|
|
def ready_to_enqueue?
|
|
!enqueued? && !installation_attempted?
|
|
end
|
|
|
|
def has_post_install_message?
|
|
!post_install_message.empty?
|
|
end
|
|
|
|
def ignorable_dependency?(dep)
|
|
dep.type == :development || dep.name == @name
|
|
end
|
|
|
|
# Checks installed dependencies against spec's dependencies to make
|
|
# sure needed dependencies have been installed.
|
|
def dependencies_installed?(all_specs)
|
|
installed_specs = all_specs.select(&:installed?).map(&:name)
|
|
dependencies.all? {|d| installed_specs.include? d.name }
|
|
end
|
|
|
|
# Represents only the non-development dependencies, the ones that are
|
|
# itself and are in the total list.
|
|
def dependencies
|
|
@dependencies ||= begin
|
|
all_dependencies.reject {|dep| ignorable_dependency? dep }
|
|
end
|
|
end
|
|
|
|
def missing_lockfile_dependencies(all_spec_names)
|
|
deps = all_dependencies.reject {|dep| ignorable_dependency? dep }
|
|
deps.reject {|dep| all_spec_names.include? dep.name }
|
|
end
|
|
|
|
# Represents all dependencies
|
|
def all_dependencies
|
|
@spec.dependencies
|
|
end
|
|
|
|
def to_s
|
|
"#<#{self.class} #{@spec.full_name} (#{state})>"
|
|
end
|
|
end
|
|
|
|
def self.call(*args)
|
|
new(*args).call
|
|
end
|
|
|
|
attr_reader :size
|
|
|
|
def initialize(installer, all_specs, size, standalone, force)
|
|
@installer = installer
|
|
@size = size
|
|
@standalone = standalone
|
|
@force = force
|
|
@specs = all_specs.map {|s| SpecInstallation.new(s) }
|
|
@spec_set = all_specs
|
|
@rake = @specs.find {|s| s.name == "rake" }
|
|
end
|
|
|
|
def call
|
|
check_for_corrupt_lockfile
|
|
|
|
if @size > 1
|
|
install_with_worker
|
|
else
|
|
install_serially
|
|
end
|
|
|
|
handle_error if failed_specs.any?
|
|
@specs
|
|
ensure
|
|
worker_pool && worker_pool.stop
|
|
end
|
|
|
|
def check_for_corrupt_lockfile
|
|
missing_dependencies = @specs.map do |s|
|
|
[
|
|
s,
|
|
s.missing_lockfile_dependencies(@specs.map(&:name)),
|
|
]
|
|
end.reject {|a| a.last.empty? }
|
|
return if missing_dependencies.empty?
|
|
|
|
warning = []
|
|
warning << "Your lockfile was created by an old Bundler that left some things out."
|
|
if @size != 1
|
|
warning << "Because of the missing DEPENDENCIES, we can only install gems one at a time, instead of installing #{@size} at a time."
|
|
@size = 1
|
|
end
|
|
warning << "You can fix this by adding the missing gems to your Gemfile, running bundle install, and then removing the gems from your Gemfile."
|
|
warning << "The missing gems are:"
|
|
|
|
missing_dependencies.each do |spec, missing|
|
|
warning << "* #{missing.map(&:name).join(", ")} depended upon by #{spec.name}"
|
|
end
|
|
|
|
Bundler.ui.warn(warning.join("\n"))
|
|
end
|
|
|
|
private
|
|
|
|
def failed_specs
|
|
@specs.select(&:failed?)
|
|
end
|
|
|
|
def install_with_worker
|
|
enqueue_specs
|
|
process_specs until finished_installing?
|
|
end
|
|
|
|
def install_serially
|
|
until finished_installing?
|
|
raise "failed to find a spec to enqueue while installing serially" unless spec_install = @specs.find(&:ready_to_enqueue?)
|
|
spec_install.state = :enqueued
|
|
do_install(spec_install, 0)
|
|
end
|
|
end
|
|
|
|
def worker_pool
|
|
@worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num|
|
|
do_install(spec_install, worker_num)
|
|
}
|
|
end
|
|
|
|
def do_install(spec_install, worker_num)
|
|
Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL, spec_install)
|
|
gem_installer = Bundler::GemInstaller.new(
|
|
spec_install.spec, @installer, @standalone, worker_num, @force
|
|
)
|
|
success, message = gem_installer.install_from_spec
|
|
if success
|
|
spec_install.state = :installed
|
|
spec_install.post_install_message = message unless message.nil?
|
|
else
|
|
spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}"
|
|
spec_install.state = :failed
|
|
end
|
|
Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install)
|
|
spec_install
|
|
end
|
|
|
|
# Dequeue a spec and save its post-install message and then enqueue the
|
|
# remaining specs.
|
|
# Some specs might've had to wait til this spec was installed to be
|
|
# processed so the call to `enqueue_specs` is important after every
|
|
# dequeue.
|
|
def process_specs
|
|
worker_pool.deq
|
|
enqueue_specs
|
|
end
|
|
|
|
def finished_installing?
|
|
@specs.all? do |spec|
|
|
return true if spec.failed?
|
|
spec.installed?
|
|
end
|
|
end
|
|
|
|
def handle_error
|
|
errors = failed_specs.map(&:error)
|
|
if exception = errors.find {|e| e.is_a?(Bundler::BundlerError) }
|
|
raise exception
|
|
end
|
|
raise Bundler::InstallError, errors.join("\n\n")
|
|
end
|
|
|
|
def require_tree_for_spec(spec)
|
|
tree = @spec_set.what_required(spec)
|
|
t = String.new("In #{File.basename(SharedHelpers.default_gemfile)}:\n")
|
|
tree.each_with_index do |s, depth|
|
|
t << " " * depth.succ << s.name
|
|
unless tree.last == s
|
|
t << %( was resolved to #{s.version}, which depends on)
|
|
end
|
|
t << %(\n)
|
|
end
|
|
t
|
|
end
|
|
|
|
# Keys in the remains hash represent uninstalled gems specs.
|
|
# We enqueue all gem specs that do not have any dependencies.
|
|
# Later we call this lambda again to install specs that depended on
|
|
# previously installed specifications. We continue until all specs
|
|
# are installed.
|
|
def enqueue_specs
|
|
@specs.select(&:ready_to_enqueue?).each do |spec|
|
|
next if @rake && !@rake.installed? && spec.name != @rake.name
|
|
|
|
if spec.dependencies_installed? @specs
|
|
spec.state = :enqueued
|
|
worker_pool.enq spec
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|