#!/usr/bin/ruby -s # -*- coding: us-ascii -*- require 'uri' require 'digest/sha1' require 'digest/sha2' require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) require File.expand_path("../lib/colorize", __FILE__) STDOUT.sync = true $srcdir ||= nil $archname = nil if ($archname ||= nil) == "" $keep_temp ||= nil $patch_file ||= nil $packages ||= nil $digests ||= nil $tooldir = File.expand_path("..", __FILE__) $unicode_version = nil if ($unicode_version ||= nil) == "" $colorize = Colorize.new def usage <<USAGE usage: #{File.basename $0} [option...] new-directory-to-save [version ...] options: -srcdir=PATH source directory path -archname=NAME make the basename of snapshots NAME -keep_temp keep temporary working directory -patch_file=PATCH apply PATCH file after export -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) -unicode_version=VER Unicode version to generate encodings -svn[=URL] make snapshot from SVN repository (#{SVNURL}) -help, --help show this message version: master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL each versions may be followed by optional @revision. USAGE end DIGESTS = %w[SHA1 SHA256 SHA512] PACKAGES = { "tar" => %w".tar", "bzip" => %w".tar.bz2 bzip2 -c", "gzip" => %w".tar.gz gzip -c", "xz" => %w".tar.xz xz -c", "zip" => %w".zip zip -Xqr", } DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} elsif gzip = ENV.delete("GZIP") PACKAGES["gzip"].concat(gzip.shellsplit) end if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2') mflags.strip! ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) end ENV["LC_ALL"] = ENV["LANG"] = "C" SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") # https git clone is disabled at git.ruby-lang.org/ruby.git. GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ ENV["VPATH"] ||= "include/ruby" YACC = ENV["YACC"] ||= "bison" ENV["BASERUBY"] ||= "ruby" ENV["RUBY"] ||= "ruby" ENV["MV"] ||= "mv" ENV["RM"] ||= "rm -f" ENV["MINIRUBY"] ||= "ruby" ENV["PROGRAM"] ||= "ruby" ENV["AUTOCONF"] ||= "autoconf" ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" ENV["TZ"] = "UTC" class String # for older ruby alias bytesize size unless method_defined?(:bytesize) end class Dir def self.mktmpdir(path) path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}") begin mkdir(path) rescue Errno::EEXIST path.succ! retry end path end unless respond_to?(:mktmpdir) end $packages &&= $packages.split(/[, ]+/).tap {|pkg| if all = pkg.index("all") pkg[all, 1] = DEFAULT_PACKAGES - pkg end pkg -= PACKAGES.keys pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}" } $packages ||= DEFAULT_PACKAGES $digests &&= $digests.split(/[, ]+/).tap {|dig| dig -= DIGESTS dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}" } $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) path = ENV["PATH"].split(File::PATH_SEPARATOR) %w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| cmd, = ENV[var].shellsplit unless path.any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end end %w[BASERUBY RUBY MINIRUBY].each do |var| %x[#{ENV[var]} --disable-gem -e1 2>&1] if $?.success? ENV[var] += ' --disable-gem' end end if defined?($help) or defined?($_help) puts usage exit end unless destdir = ARGV.shift abort usage end revisions = ARGV.empty? ? [nil] : ARGV if $exported abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" end FileUtils.mkpath(destdir) destdir = File.expand_path(destdir) tmp = Dir.mktmpdir("ruby-snapshot") FileUtils.mkpath(tmp) at_exit { Dir.chdir "/" FileUtils.rm_rf(tmp) } unless $keep_temp def tar_create(tarball, dir) require 'rubygems' require 'rubygems/package' require 'rubygems/package/tar_writer' header = Gem::Package::TarHeader dir_type = "5" uname = gname = "ruby" File.open(tarball, "wb") do |f| w = Gem::Package::TarWriter.new(f) list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) list.reject! {|name| name.end_with?("/.")} list.sort_by! {|name| name.split("/")} list.each do |path| next if File.basename(path) == "." s = File.stat(path) mode = 0644 case when s.file? type = nil size = s.size mode |= 0111 if s.executable? when s.directory? path += "/" type = dir_type size = 0 mode |= 0111 else next end name, prefix = w.split_name(path) h = header.new(name: name, prefix: prefix, typeflag: type, mode: mode, size: size, mtime: s.mtime, uname: uname, gname: gname) f.write(h) if size > 0 IO.copy_stream(path, f) f.write("\0" * (-size % 512)) end end end true rescue => e warn e.message false end def touch_all(time, pattern, opt, &cond) Dir.glob(pattern, opt) do |n| stat = File.stat(n) if stat.file? or stat.directory? next if cond and !yield(n, stat) File.utime(time, time, n) end end rescue false else true end class MAKE < Struct.new(:prog, :args) def initialize(vars) vars = vars.map {|arg| arg.join("=")} super(ENV["MAKE"] || ENV["make"] || "make", vars) end def run(target) err = IO.pipe do |r, w| begin pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close}) w.close r.read ensure Process.wait(pid) end end if $?.success? true else STDERR.puts err $colorize.fail("#{target} failed") false end end end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` end case rev when nil url = nil when /\A(?:master|trunk)\z/ url = vcs.trunk when /\Abranches\// url = vcs.branch($') when /\Atags\// url = vcs.tag($') when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") when /\A(.*)\.(.*)\.(.*)-p(\d+)/ patchlevel = true tag = "p#{$4}" url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") patchlevel = true tag = "" url = vcs.tag("v#{$1}_#{$2}_#{$3}") else url = vcs.branch("ruby_#{rev.tr('.', '_')}") end else warn "#{$0}: unknown version - #{rev}" return end if info = vcs.get_revisions(url) modified = info[2] else modified = Time.now - 10 end if !revision and info revision = info url ||= vcs.branch(revision[3]) revision = revision[1] end version = nil unless revision url = vcs.trunk vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1} unless rev == version warn "#{$0}: #{rev} not found" return end revision = vcs.get_revisions(url)[1] end v = "ruby" puts "Exporting #{rev}@#{revision}" exported = tmp ? File.join(tmp, v) : v unless vcs.export(revision, url, exported, true) {|line| print line} warn("Export failed") return end if $srcdir Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| puts "copying #{file}" dest = exported + file[$srcdir.size..-1] FileUtils.mkpath(File.dirname(dest)) begin FileUtils.cp_r(file, dest) FileUtils.chmod_R("a+rwX,go-w", dest) rescue SystemCallError end end end status = IO.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) v = Dir.glob("ruby-*").select(&File.method(:directory?)) v.size == 1 or abort "#{File.basename $0}: not exported" v = v[0] end open("#{v}/revision.h", "wb") {|f| short = vcs.short_revision(revision) f.puts "#define RUBY_REVISION #{short.inspect}" unless short == revision f.puts "#define RUBY_FULL_REVISION #{revision.inspect}" end } version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return if patchlevel unless tag.empty? versionhdr ||= IO.read("#{v}/version.h") patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) end elsif prerelease versionhdr ||= IO.read("#{v}/version.h") versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) IO.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end if $archname n = $archname elsif tag.empty? n = "ruby-#{version}" else n = "ruby-#{version}-#{tag}" end File.directory?(n) or File.rename v, n v = n if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}") puts $colorize.fail("patching failed") return end def (clean = []).add(n) push(n); n end Dir.chdir(v) do unless File.exist?("ChangeLog") vcs.export_changelog(url, nil, revision, "ChangeLog") end unless touch_all(modified, "**/*", File::FNM_DOTMATCH) modified = nil colors = %w[red yellow green cyan blue magenta] "take a breath, and go ahead".scan(/./) do |c| if c == ' ' print c else colors.push(color = colors.shift) print $colorize.decorate(c, color) end sleep(c == "," ? 0.7 : 0.05) end puts end File.open(clean.add("cross.rb"), "w") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" f.puts "RUBY_PLATFORM='none'" f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" f.puts "RUBY_VERSION='#{version}'" end unless File.exist?("configure") print "creating configure..." unless system([ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end puts $colorize.pass(" done") end clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") puts extout = clean.add('tmp') begin status = IO.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end File.open(clean.add("config.status"), "w") {|f| f.print status } FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") File.open("#{hdrdir}/config.h", "w") {} FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") File.open("#{defaults}/operating_system.rb", "w") {} File.open("#{defaults}/ruby.rb", "w") {} miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, "PATH_SEPARATOR"=>File::PATH_SEPARATOR, "MINIRUBY"=>miniruby, "RUBY"=>ENV["RUBY"], "BASERUBY"=>baseruby, "PWD"=>Dir.pwd, "ruby_version"=>version, "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 end vars.delete("UNICODE_FILES") # for stable branches vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' update-download:: touch-unicode-files prepare-package: prereq after-update clean-cache: $(CLEAN_CACHE) after-update:: extract-gems extract-gems: update-gems update-gems: $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: touch-unicode-files: APPEND open(clean.add("Makefile"), "w") do |f| f.puts mk end File.open(clean.add("revision.tmp"), "w") {} File.open(clean.add(".revision.time"), "w") {} ENV["CACHE_SAVE"] = "no" make = MAKE.new(args) return unless make.run("update-download") clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time") Dir.glob("**/*") do |dest| next unless File.symlink?(dest) orig = File.expand_path(File.readlink(dest), File.dirname(dest)) File.unlink(dest) FileUtils.cp_r(orig, dest) end File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"])) return unless make.run("prepare-package") return unless make.run("clean-cache") if modified new_time = modified + 2 touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat| stat.mtime > modified unless clean.include?(name) end modified = new_time end print "prerequisites" else system(*%W"#{YACC} -o parse.c parse.y") end vcs.after_export(".") if exported FileUtils.rm_rf(clean) unless $keep_temp FileUtils.rm_rf(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| next if /^\s*(?:#|$)/ =~ line name, version, _ = line.split(' ') "gems/#{name}-#{version}.gem" } FileUtils.rm_f(gems) else FileUtils.rm_rf("gems") end if modified touch_all(modified, "**/*/", 0) do |name, stat| stat.mtime > modified end File.utime(modified, modified, ".") end unless $?.success? puts $colorize.fail(" failed") return end puts $colorize.pass(" done") end if v == "." v = File.basename(Dir.pwd) Dir.chdir ".." else Dir.chdir(File.dirname(v)) v = File.basename(v) end tarball = nil return $packages.collect do |mesg| (ext, *cmd) = PACKAGES[mesg] File.directory?(destdir) or FileUtils.mkpath(destdir) file = File.join(destdir, "#{$archname||v}#{ext}") case ext when /\.tar/ if tarball next if tarball.empty? else tarball = ext == ".tar" ? file : "#{$archname||v}.tar" print "creating tarball... #{tarball}" if tar_create(tarball, v) puts $colorize.pass(" done") File.utime(modified, modified, tarball) if modified next if tarball == file else puts $colorize.fail(" failed") tarball = "" next end end print "creating #{mesg} tarball... #{file}" done = system(*cmd, tarball, out: file) else print "creating #{mesg} archive... #{file}" if Hash === cmd.last *cmd, opt = *cmd cmd << file << v << opt else (cmd = cmd.dup) << file << v end done = system(*cmd) end if done puts $colorize.pass(" done") file else puts $colorize.fail(" failed") nil end end.compact ensure FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp Dir.chdir(pwd) end if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" end if $srcdir vcs = VCS.detect($srcdir) elsif $svn vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) elsif $git abort "#{File.basename $0}: use -srcdir with cloned local repository" else begin vcs = VCS.detect(File.expand_path("../..", __FILE__)) rescue VCS::NotFoundError vcs = VCS::SVN.new(SVNURL) end end release_date = Time.now.getutc info = {} success = true revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| if !name success = false next end str = open(name, "rb") {|f| f.read} pathname = Pathname(name) basename = pathname.basename.to_s extname = pathname.extname.sub(/\A\./, '') version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1] key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version info[key]['date'] = release_date.strftime('%Y-%m-%d') if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" else info[key]['filename'][extname] = basename end info[key]['size'][extname] = str.bytesize puts "* #{$colorize.pass(name)}" puts " SIZE: #{str.bytesize} bytes" $digests.each do |alg| digest = Digest(alg).hexdigest(str) info[key][alg.downcase][extname] = digest printf " %-8s%s\n", "#{alg}:", digest end end yaml = info.values.to_yaml json = info.values.to_json puts "#{$colorize.pass('YAML:')}" puts yaml puts "#{$colorize.pass('JSON:')}" puts json infodir = Pathname(destdir) + 'info' infodir.mkpath (infodir+'info.yml').write(yaml) (infodir+'info.json').write(json) exit false if !success # vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix