mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
56809537a4
While generate_index did clean up temporary directory, when running with --update flag, that did not happen and the temporary directory was left behind. This commit fixes that and modifies tests in order to make sure this is not reintroduced later on. Fixes #5635. https://github.com/rubygems/rubygems/commit/9fa34dc329
427 lines
11 KiB
Ruby
427 lines
11 KiB
Ruby
# frozen_string_literal: true
|
|
require_relative '../rubygems'
|
|
require_relative 'package'
|
|
require 'tmpdir'
|
|
|
|
##
|
|
# Top level class for building the gem repository index.
|
|
|
|
class Gem::Indexer
|
|
include Gem::UserInteraction
|
|
|
|
##
|
|
# Build indexes for RubyGems 1.2.0 and newer when true
|
|
|
|
attr_accessor :build_modern
|
|
|
|
##
|
|
# Index install location
|
|
|
|
attr_reader :dest_directory
|
|
|
|
##
|
|
# Specs index install location
|
|
|
|
attr_reader :dest_specs_index
|
|
|
|
##
|
|
# Latest specs index install location
|
|
|
|
attr_reader :dest_latest_specs_index
|
|
|
|
##
|
|
# Prerelease specs index install location
|
|
|
|
attr_reader :dest_prerelease_specs_index
|
|
|
|
##
|
|
# Index build directory
|
|
|
|
attr_reader :directory
|
|
|
|
##
|
|
# Create an indexer that will index the gems in +directory+.
|
|
|
|
def initialize(directory, options = {})
|
|
require 'fileutils'
|
|
require 'tmpdir'
|
|
require 'zlib'
|
|
|
|
options = { :build_modern => true }.merge options
|
|
|
|
@build_modern = options[:build_modern]
|
|
|
|
@dest_directory = directory
|
|
@directory = Dir.mktmpdir 'gem_generate_index'
|
|
|
|
marshal_name = "Marshal.#{Gem.marshal_version}"
|
|
|
|
@master_index = File.join @directory, 'yaml'
|
|
@marshal_index = File.join @directory, marshal_name
|
|
|
|
@quick_dir = File.join @directory, 'quick'
|
|
@quick_marshal_dir = File.join @quick_dir, marshal_name
|
|
@quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH
|
|
|
|
@quick_index = File.join @quick_dir, 'index'
|
|
@latest_index = File.join @quick_dir, 'latest_index'
|
|
|
|
@specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
|
|
@latest_specs_index =
|
|
File.join(@directory, "latest_specs.#{Gem.marshal_version}")
|
|
@prerelease_specs_index =
|
|
File.join(@directory, "prerelease_specs.#{Gem.marshal_version}")
|
|
@dest_specs_index =
|
|
File.join(@dest_directory, "specs.#{Gem.marshal_version}")
|
|
@dest_latest_specs_index =
|
|
File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}")
|
|
@dest_prerelease_specs_index =
|
|
File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}")
|
|
|
|
@files = []
|
|
end
|
|
|
|
##
|
|
# Build various indices
|
|
|
|
def build_indices
|
|
specs = map_gems_to_specs gem_file_list
|
|
Gem::Specification._resort! specs
|
|
build_marshal_gemspecs specs
|
|
build_modern_indices specs if @build_modern
|
|
|
|
compress_indices
|
|
end
|
|
|
|
##
|
|
# Builds Marshal quick index gemspecs.
|
|
|
|
def build_marshal_gemspecs(specs)
|
|
count = specs.count
|
|
progress = ui.progress_reporter count,
|
|
"Generating Marshal quick index gemspecs for #{count} gems",
|
|
"Complete"
|
|
|
|
files = []
|
|
|
|
Gem.time 'Generated Marshal quick index gemspecs' do
|
|
specs.each do |spec|
|
|
next if spec.default_gem?
|
|
spec_file_name = "#{spec.original_name}.gemspec.rz"
|
|
marshal_name = File.join @quick_marshal_dir, spec_file_name
|
|
|
|
marshal_zipped = Gem.deflate Marshal.dump(spec)
|
|
|
|
File.open marshal_name, 'wb' do |io|
|
|
io.write marshal_zipped
|
|
end
|
|
|
|
files << marshal_name
|
|
|
|
progress.updated spec.original_name
|
|
end
|
|
|
|
progress.done
|
|
end
|
|
|
|
@files << @quick_marshal_dir
|
|
|
|
files
|
|
end
|
|
|
|
##
|
|
# Build a single index for RubyGems 1.2 and newer
|
|
|
|
def build_modern_index(index, file, name)
|
|
say "Generating #{name} index"
|
|
|
|
Gem.time "Generated #{name} index" do
|
|
File.open(file, 'wb') do |io|
|
|
specs = index.map do |*spec|
|
|
# We have to splat here because latest_specs is an array, while the
|
|
# others are hashes.
|
|
spec = spec.flatten.last
|
|
platform = spec.original_platform
|
|
|
|
# win32-api-1.0.4-x86-mswin32-60
|
|
unless String === platform
|
|
alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
|
|
next
|
|
end
|
|
|
|
platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
|
|
[spec.name, spec.version, platform]
|
|
end
|
|
|
|
specs = compact_specs(specs)
|
|
Marshal.dump(specs, io)
|
|
end
|
|
end
|
|
end
|
|
|
|
##
|
|
# Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease
|
|
|
|
def build_modern_indices(specs)
|
|
prerelease, released = specs.partition do |s|
|
|
s.version.prerelease?
|
|
end
|
|
latest_specs =
|
|
Gem::Specification._latest_specs specs
|
|
|
|
build_modern_index(released.sort, @specs_index, 'specs')
|
|
build_modern_index(latest_specs.sort, @latest_specs_index, 'latest specs')
|
|
build_modern_index(prerelease.sort, @prerelease_specs_index,
|
|
'prerelease specs')
|
|
|
|
@files += [@specs_index,
|
|
"#{@specs_index}.gz",
|
|
@latest_specs_index,
|
|
"#{@latest_specs_index}.gz",
|
|
@prerelease_specs_index,
|
|
"#{@prerelease_specs_index}.gz"]
|
|
end
|
|
|
|
def map_gems_to_specs(gems)
|
|
gems.map do |gemfile|
|
|
if File.size(gemfile) == 0
|
|
alert_warning "Skipping zero-length gem: #{gemfile}"
|
|
next
|
|
end
|
|
|
|
begin
|
|
spec = Gem::Package.new(gemfile).spec
|
|
spec.loaded_from = gemfile
|
|
|
|
spec.abbreviate
|
|
spec.sanitize
|
|
|
|
spec
|
|
rescue SignalException
|
|
alert_error "Received signal, exiting"
|
|
raise
|
|
rescue Exception => e
|
|
msg = ["Unable to process #{gemfile}",
|
|
"#{e.message} (#{e.class})",
|
|
"\t#{e.backtrace.join "\n\t"}"].join("\n")
|
|
alert_error msg
|
|
end
|
|
end.compact
|
|
end
|
|
|
|
##
|
|
# Compresses indices on disk
|
|
#--
|
|
# All future files should be compressed using gzip, not deflate
|
|
|
|
def compress_indices
|
|
say "Compressing indices"
|
|
|
|
Gem.time 'Compressed indices' do
|
|
if @build_modern
|
|
gzip @specs_index
|
|
gzip @latest_specs_index
|
|
gzip @prerelease_specs_index
|
|
end
|
|
end
|
|
end
|
|
|
|
##
|
|
# Compacts Marshal output for the specs index data source by using identical
|
|
# objects as much as possible.
|
|
|
|
def compact_specs(specs)
|
|
names = {}
|
|
versions = {}
|
|
platforms = {}
|
|
|
|
specs.map do |(name, version, platform)|
|
|
names[name] = name unless names.include? name
|
|
versions[version] = version unless versions.include? version
|
|
platforms[platform] = platform unless platforms.include? platform
|
|
|
|
[names[name], versions[version], platforms[platform]]
|
|
end
|
|
end
|
|
|
|
##
|
|
# Compress +filename+ with +extension+.
|
|
|
|
def compress(filename, extension)
|
|
data = Gem.read_binary filename
|
|
|
|
zipped = Gem.deflate data
|
|
|
|
File.open "#{filename}.#{extension}", 'wb' do |io|
|
|
io.write zipped
|
|
end
|
|
end
|
|
|
|
##
|
|
# List of gem file names to index.
|
|
|
|
def gem_file_list
|
|
Gem::Util.glob_files_in_dir("*.gem", File.join(@dest_directory, "gems"))
|
|
end
|
|
|
|
##
|
|
# Builds and installs indices.
|
|
|
|
def generate_index
|
|
make_temp_directories
|
|
build_indices
|
|
install_indices
|
|
rescue SignalException
|
|
ensure
|
|
FileUtils.rm_rf @directory
|
|
end
|
|
|
|
##
|
|
# Zlib::GzipWriter wrapper that gzips +filename+ on disk.
|
|
|
|
def gzip(filename)
|
|
Zlib::GzipWriter.open "#{filename}.gz" do |io|
|
|
io.write Gem.read_binary(filename)
|
|
end
|
|
end
|
|
|
|
##
|
|
# Install generated indices into the destination directory.
|
|
|
|
def install_indices
|
|
verbose = Gem.configuration.really_verbose
|
|
|
|
say "Moving index into production dir #{@dest_directory}" if verbose
|
|
|
|
files = @files
|
|
files.delete @quick_marshal_dir if files.include? @quick_dir
|
|
|
|
if files.include? @quick_marshal_dir and not files.include? @quick_dir
|
|
files.delete @quick_marshal_dir
|
|
|
|
dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
|
|
|
|
FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose
|
|
FileUtils.rm_rf dst_name, :verbose => verbose
|
|
FileUtils.mv(@quick_marshal_dir, dst_name,
|
|
:verbose => verbose, :force => true)
|
|
end
|
|
|
|
files = files.map do |path|
|
|
path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
|
|
end
|
|
|
|
files.each do |file|
|
|
src_name = File.join @directory, file
|
|
dst_name = File.join @dest_directory, file
|
|
|
|
FileUtils.rm_rf dst_name, :verbose => verbose
|
|
FileUtils.mv(src_name, @dest_directory,
|
|
:verbose => verbose, :force => true)
|
|
end
|
|
end
|
|
|
|
##
|
|
# Make directories for index generation
|
|
|
|
def make_temp_directories
|
|
FileUtils.rm_rf @directory
|
|
FileUtils.mkdir_p @directory, :mode => 0700
|
|
FileUtils.mkdir_p @quick_marshal_dir
|
|
end
|
|
|
|
##
|
|
# Ensure +path+ and path with +extension+ are identical.
|
|
|
|
def paranoid(path, extension)
|
|
data = Gem.read_binary path
|
|
compressed_data = Gem.read_binary "#{path}.#{extension}"
|
|
|
|
unless data == Gem::Util.inflate(compressed_data)
|
|
raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
|
|
end
|
|
end
|
|
|
|
##
|
|
# Perform an in-place update of the repository from newly added gems.
|
|
|
|
def update_index
|
|
make_temp_directories
|
|
|
|
specs_mtime = File.stat(@dest_specs_index).mtime
|
|
newest_mtime = Time.at 0
|
|
|
|
updated_gems = gem_file_list.select do |gem|
|
|
gem_mtime = File.stat(gem).mtime
|
|
newest_mtime = gem_mtime if gem_mtime > newest_mtime
|
|
gem_mtime >= specs_mtime
|
|
end
|
|
|
|
if updated_gems.empty?
|
|
say 'No new gems'
|
|
terminate_interaction 0
|
|
end
|
|
|
|
specs = map_gems_to_specs updated_gems
|
|
prerelease, released = specs.partition {|s| s.version.prerelease? }
|
|
|
|
files = build_marshal_gemspecs specs
|
|
|
|
Gem.time 'Updated indexes' do
|
|
update_specs_index released, @dest_specs_index, @specs_index
|
|
update_specs_index released, @dest_latest_specs_index, @latest_specs_index
|
|
update_specs_index(prerelease,
|
|
@dest_prerelease_specs_index,
|
|
@prerelease_specs_index)
|
|
end
|
|
|
|
compress_indices
|
|
|
|
verbose = Gem.configuration.really_verbose
|
|
|
|
say "Updating production dir #{@dest_directory}" if verbose
|
|
|
|
files << @specs_index
|
|
files << "#{@specs_index}.gz"
|
|
files << @latest_specs_index
|
|
files << "#{@latest_specs_index}.gz"
|
|
files << @prerelease_specs_index
|
|
files << "#{@prerelease_specs_index}.gz"
|
|
|
|
files = files.map do |path|
|
|
path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
|
|
end
|
|
|
|
files.each do |file|
|
|
src_name = File.join @directory, file
|
|
dst_name = File.join @dest_directory, file # REFACTOR: duped above
|
|
|
|
FileUtils.mv src_name, dst_name, :verbose => verbose,
|
|
:force => true
|
|
|
|
File.utime newest_mtime, newest_mtime, dst_name
|
|
end
|
|
ensure
|
|
FileUtils.rm_rf @directory
|
|
end
|
|
|
|
##
|
|
# Combines specs in +index+ and +source+ then writes out a new copy to
|
|
# +dest+. For a latest index, does not ensure the new file is minimal.
|
|
|
|
def update_specs_index(index, source, dest)
|
|
specs_index = Marshal.load Gem.read_binary(source)
|
|
|
|
index.each do |spec|
|
|
platform = spec.original_platform
|
|
platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
|
|
specs_index << [spec.name, spec.version, platform]
|
|
end
|
|
|
|
specs_index = compact_specs specs_index.uniq.sort
|
|
|
|
File.open dest, 'wb' do |io|
|
|
Marshal.dump specs_index, io
|
|
end
|
|
end
|
|
end
|