2018-11-02 19:07:56 -04:00
# frozen_string_literal: true
2019-06-01 05:49:40 -04:00
require_relative " ../worker "
require_relative " gem_installer "
2018-11-02 19:07:56 -04:00
module Bundler
class ParallelInstaller
class SpecInstallation
2021-04-14 23:47:04 -04:00
attr_accessor :spec , :name , :full_name , :post_install_message , :state , :error
2018-11-02 19:07:56 -04:00
def initialize ( spec )
@spec = spec
@name = spec . name
2021-04-14 23:47:04 -04:00
@full_name = spec . full_name
2018-11-02 19:07:56 -04:00
@state = :none
@post_install_message = " "
@error = nil
end
def installed?
state == :installed
end
def enqueued?
state == :enqueued
end
def failed?
state == :failed
end
def ready_to_enqueue?
2021-04-14 23:47:04 -04:00
state == :none
2018-11-02 19:07:56 -04:00
end
def has_post_install_message?
! post_install_message . empty?
end
def ignorable_dependency? ( dep )
dep . type == :development || dep . name == @name
end
# Checks installed dependencies against spec's dependencies to make
# sure needed dependencies have been installed.
def dependencies_installed? ( all_specs )
installed_specs = all_specs . select ( & :installed? ) . map ( & :name )
dependencies . all? { | d | installed_specs . include? d . name }
end
# Represents only the non-development dependencies, the ones that are
# itself and are in the total list.
def dependencies
2021-04-14 23:47:04 -04:00
@dependencies || = all_dependencies . reject { | dep | ignorable_dependency? dep }
2018-11-02 19:07:56 -04:00
end
def missing_lockfile_dependencies ( all_spec_names )
2021-04-14 23:47:04 -04:00
dependencies . reject { | dep | all_spec_names . include? dep . name }
2018-11-02 19:07:56 -04:00
end
# Represents all dependencies
def all_dependencies
@spec . dependencies
end
def to_s
2021-04-14 23:47:04 -04:00
" # < #{ self . class } #{ full_name } ( #{ state } )> "
2018-11-02 19:07:56 -04:00
end
end
def self . call ( * args )
new ( * args ) . call
end
attr_reader :size
def initialize ( installer , all_specs , size , standalone , force )
@installer = installer
@size = size
@standalone = standalone
@force = force
@specs = all_specs . map { | s | SpecInstallation . new ( s ) }
@spec_set = all_specs
@rake = @specs . find { | s | s . name == " rake " }
end
def call
check_for_corrupt_lockfile
2021-04-14 23:47:04 -04:00
if @rake
do_install ( @rake , 0 )
Gem :: Specification . reset
end
2018-11-02 19:07:56 -04:00
if @size > 1
install_with_worker
else
install_serially
end
2021-04-14 23:47:04 -04:00
check_for_unmet_dependencies
2020-05-08 01:19:04 -04:00
handle_error if failed_specs . any?
2018-11-02 19:07:56 -04:00
@specs
ensure
worker_pool && worker_pool . stop
end
2021-04-14 23:47:04 -04:00
def check_for_unmet_dependencies
unmet_dependencies = @specs . map do | s |
[
s ,
s . dependencies . reject { | dep | @specs . any? { | spec | dep . matches_spec? ( spec . spec ) } } ,
]
end . reject { | a | a . last . empty? }
return if unmet_dependencies . empty?
warning = [ ]
warning << " Your lockfile doesn't include a valid resolution. "
warning << " You can fix this by regenerating your lockfile or trying to manually editing the bad locked gems to a version that satisfies all dependencies. "
warning << " The unmet dependencies are: "
unmet_dependencies . each do | spec , unmet_spec_dependencies |
unmet_spec_dependencies . each do | unmet_spec_dependency |
warning << " * #{ unmet_spec_dependency } , depended upon #{ spec . full_name } , unsatisfied by #{ @specs . find { | s | s . name == unmet_spec_dependency . name && ! unmet_spec_dependency . matches_spec? ( s . spec ) } . full_name } "
end
end
Bundler . ui . warn ( warning . join ( " \n " ) )
end
2018-11-02 19:07:56 -04:00
def check_for_corrupt_lockfile
missing_dependencies = @specs . map do | s |
[
s ,
s . missing_lockfile_dependencies ( @specs . map ( & :name ) ) ,
]
2019-04-14 02:01:35 -04:00
end . reject { | a | a . last . empty? }
2018-11-02 19:07:56 -04:00
return if missing_dependencies . empty?
warning = [ ]
warning << " Your lockfile was created by an old Bundler that left some things out. "
if @size != 1
warning << " Because of the missing DEPENDENCIES, we can only install gems one at a time, instead of installing #{ @size } at a time. "
@size = 1
end
warning << " You can fix this by adding the missing gems to your Gemfile, running bundle install, and then removing the gems from your Gemfile. "
warning << " The missing gems are: "
missing_dependencies . each do | spec , missing |
warning << " * #{ missing . map ( & :name ) . join ( " , " ) } depended upon by #{ spec . name } "
end
Bundler . ui . warn ( warning . join ( " \n " ) )
end
2020-10-15 00:20:25 -04:00
private
2018-11-02 19:07:56 -04:00
2020-05-08 01:19:04 -04:00
def failed_specs
@specs . select ( & :failed? )
end
2018-11-02 19:07:56 -04:00
def install_with_worker
enqueue_specs
process_specs until finished_installing?
end
def install_serially
until finished_installing?
raise " failed to find a spec to enqueue while installing serially " unless spec_install = @specs . find ( & :ready_to_enqueue? )
spec_install . state = :enqueued
do_install ( spec_install , 0 )
end
end
def worker_pool
2019-04-14 02:01:35 -04:00
@worker_pool || = Bundler :: Worker . new @size , " Parallel Installer " , lambda { | spec_install , worker_num |
2018-11-02 19:07:56 -04:00
do_install ( spec_install , worker_num )
}
end
def do_install ( spec_install , worker_num )
Plugin . hook ( Plugin :: Events :: GEM_BEFORE_INSTALL , spec_install )
gem_installer = Bundler :: GemInstaller . new (
spec_install . spec , @installer , @standalone , worker_num , @force
)
2020-05-08 01:19:04 -04:00
success , message = gem_installer . install_from_spec
2018-11-02 19:07:56 -04:00
if success
spec_install . state = :installed
spec_install . post_install_message = message unless message . nil?
else
spec_install . error = " #{ message } \n \n #{ require_tree_for_spec ( spec_install . spec ) } "
[rubygems/rubygems] Fix parallel installer race condition
The main thread may detect that installation has finished and thus
abort, while the thread responsible for installing the spec that failed
has not yet set the error message.
In this case, the install process will abort with a misterious "empty"
exception. I can be force the issue to reproduce by applying the
following patch:
```diff
diff --git a/bundler/lib/bundler/installer/parallel_installer.rb b/bundler/lib/bundler/installer/parallel_installer.rb
index 3dee9f4664..7827a11d11 100644
--- a/bundler/lib/bundler/installer/parallel_installer.rb
+++ b/bundler/lib/bundler/installer/parallel_installer.rb
@@ -166,6 +166,7 @@ module Bundler
spec_install.post_install_message = message unless message.nil?
else
spec_install.state = :failed
+ sleep 1
spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}"
end
Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install)
@@ -183,6 +184,7 @@ module Bundler
end
def finished_installing?
+ sleep 0.5
@specs.all? do |spec|
return true if spec.failed?
spec.installed?
diff --git a/bundler/lib/bundler/rubygems_gem_installer.rb b/bundler/lib/bundler/rubygems_gem_installer.rb
index 8ce33c3953..c585cd517b 100644
--- a/bundler/lib/bundler/rubygems_gem_installer.rb
+++ b/bundler/lib/bundler/rubygems_gem_installer.rb
@@ -42,6 +42,7 @@ module Bundler
return true unless source = @package.instance_variable_get(:@gem)
return true unless source.respond_to?(:with_read_io)
digest = source.with_read_io do |io|
+ raise BundlerError, "asdafss"
digest = SharedHelpers.digest(:SHA256).new
digest << io.read(16_384) until io.eof?
io.rewind
```
and running `bin/rspec spec/install/gems/compact_index_spec.rb:892` will
result in
```
Run options:
include {:locations=>{"./spec/install/gems/compact_index_spec.rb"=>[892]}}
exclude {:jruby=>true, :readline=>false, :permissions=>false, :no_color_tty=>false, :ruby_repo=>false, :bundler=>"!= 2", :git=>"!= 2.26.2", :rubygems=>"!= 3.2.0.pre1", :realworld=>true, :sudo=>true}
Randomized with seed 59277
F
Retried examples: 0
Failures:
1) compact index api checksum validation raises when the checksum is the wrong length
Failure/Error: expect(err).to include("The given checksum for rack-1.0.0 (\"checksum!\") is not a valid SHA256 hexdigest nor base64digest")
expected "" to include "The given checksum for rack-1.0.0 (\"checksum!\") is not a valid SHA256 hexdigest nor base64digest"
Commands:
$ /home/deivid/.rbenv/versions/2.7.1/bin/ruby -I/home/deivid/Code/rubygems/rubygems/bundler/spec -r/home/deivid/Code/rubygems/rubygems/bundler/spec/support/artifice/compact_index_wrong_gem_checksum.rb -r/home/deivid/Code/rubygems/rubygems/bundler/spec/support/hax.rb /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle install --verbose --retry 0
Running `bundle install --retry 0 --verbose` with bundler 2.2.0.dev
Found changes from the lockfile, re-resolving dependencies because the list of sources changed, the dependencies in your gemfile changed, you added a new platform to your gemfile
HTTP GET http://localgemserver.test/versions
HTTP 200 OK http://localgemserver.test/versions
Fetching gem metadata from http://localgemserver.test/
Looking up gems ["rack"]
HTTP GET http://localgemserver.test/info/rack
HTTP 200 OK http://localgemserver.test/info/rack
Resolving dependencies...
Using bundler 2.2.0.dev
0: bundler (2.2.0.dev) from /home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/specifications/bundler-2.2.0.dev.gemspec
Fetching rack 1.0.0
Installing rack 1.0.0
Bundler::InstallError:
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:199:in `handle_error'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:102:in `call'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer/parallel_installer.rb:78:in `call'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:271:in `install_in_parallel'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:197:in `install'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:92:in `block in run'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:12:in `block in lock'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:9:in `open'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/process_lock.rb:9:in `lock'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:73:in `run'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/installer.rb:25:in `install'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli/install.rb:66:in `run'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:261:in `block in install'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/settings.rb:121:in `temporary'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:260:in `install'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/invocation.rb:127:in `invoke_command'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor.rb:399:in `dispatch'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:30:in `dispatch'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/vendor/thor/lib/thor/base.rb:476:in `start'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/cli.rb:24:in `start'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/exe/bundle:49:in `block in <top (required)>'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/lib/bundler/friendly_errors.rb:117:in `with_friendly_errors'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/gems/bundler-2.2.0.dev/exe/bundle:37:in `<top (required)>'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle:23:in `load'
/home/deivid/Code/rubygems/rubygems/bundler/tmp/1/gems/system/bin/bundle:23:in `<main>'
# $? => 5
# ./spec/install/gems/compact_index_spec.rb:892:in `block (3 levels) in <top (required)>'
# ./spec/spec_helper.rb:104:in `block (4 levels) in <top (required)>'
# ./spec/spec_helper.rb:104:in `block (3 levels) in <top (required)>'
# ./spec/support/helpers.rb:352:in `block in with_gem_path_as'
# ./spec/support/helpers.rb:366:in `without_env_side_effects'
# ./spec/support/helpers.rb:348:in `with_gem_path_as'
# ./spec/spec_helper.rb:101:in `block (2 levels) in <top (required)>'
# ./spec/spec_helper.rb:73:in `block (2 levels) in <top (required)>'
# ./spec/support/rubygems_ext.rb:90:in `load'
# ./spec/support/rubygems_ext.rb:90:in `gem_load_and_activate'
# ./spec/support/rubygems_ext.rb:18:in `gem_load'
Finished in 3.01 seconds (files took 0.14209 seconds to load)
1 example, 1 failure
Failed examples:
rspec ./spec/install/gems/compact_index_spec.rb:886 # compact index api checksum validation raises when the checksum is the wrong length
Randomized with seed 59277
```
Without any mention to `BundlerError` and the original "asdafss" message.
Fix the issue by making sure the error message is set before the ":failed"
status.
https://github.com/rubygems/rubygems/commit/83c8feb2c4
2020-05-30 11:37:56 -04:00
spec_install . state = :failed
2018-11-02 19:07:56 -04:00
end
Plugin . hook ( Plugin :: Events :: GEM_AFTER_INSTALL , spec_install )
spec_install
end
# Dequeue a spec and save its post-install message and then enqueue the
# remaining specs.
# Some specs might've had to wait til this spec was installed to be
# processed so the call to `enqueue_specs` is important after every
# dequeue.
def process_specs
worker_pool . deq
enqueue_specs
end
def finished_installing?
@specs . all? do | spec |
return true if spec . failed?
spec . installed?
end
end
def handle_error
2020-05-08 01:19:04 -04:00
errors = failed_specs . map ( & :error )
2018-11-02 19:07:56 -04:00
if exception = errors . find { | e | e . is_a? ( Bundler :: BundlerError ) }
raise exception
end
2020-05-08 01:19:04 -04:00
raise Bundler :: InstallError , errors . join ( " \n \n " )
2018-11-02 19:07:56 -04:00
end
def require_tree_for_spec ( spec )
tree = @spec_set . what_required ( spec )
t = String . new ( " In #{ File . basename ( SharedHelpers . default_gemfile ) } : \n " )
tree . each_with_index do | s , depth |
t << " " * depth . succ << s . name
unless tree . last == s
t << %( was resolved to #{ s . version } , which depends on )
end
t << %( \n )
end
t
end
# Keys in the remains hash represent uninstalled gems specs.
# We enqueue all gem specs that do not have any dependencies.
# Later we call this lambda again to install specs that depended on
# previously installed specifications. We continue until all specs
# are installed.
def enqueue_specs
@specs . select ( & :ready_to_enqueue? ) . each do | spec |
if spec . dependencies_installed? @specs
spec . state = :enqueued
worker_pool . enq spec
end
end
end
end
end