extlibs.rb: external libraries

* tool/extlibs.rb: download, extract, and patch external libraries.

* ext/fiddle/extlibs: libffi-3.2.1 and patch for mswin.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@48935 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

common.mk

@@ -900,7 +900,7 @@ dist:
 up::
 	-$(Q)$(MAKE) $(MFLAGS) REVISION_FORCE=PHONY "$(REVISION_H)"
 
-after-update:: update-unicode update-gems common-srcs
+after-update:: update-unicode update-gems common-srcs extract-extlibs
 
 update-config_files: PHONY
 	$(Q) $(BASERUBY) -C "$(srcdir)/tool" \
@@ -958,6 +958,17 @@ $(srcdir)/$(HAVE_BASERUBY:yes=lib/unicode_normalize/tables.rb): \
 		$(srcdir)/template/unicode_norm_gen.tmpl \
 		enc/unicode/data/$(UNICODE_VERSION) lib/unicode_normalize
 
+download-extlibs:
+	$(Q) $(BASERUBY) -C $(srcdir) -w tool/extlibs.rb --download ext
+
+extract-extlibs:
+	$(Q) $(BASERUBY) -C $(srcdir) -w tool/extlibs.rb --all ext
+
+clean-extlibs:
+	$(Q) $(RMALL) $(srcdir)/.downloaded-cache
+
+CLEAN_CACHE = clean-extlibs
+
 info: info-program info-libruby_a info-libruby_so info-arch
 info-program: PHONY
 	@echo PROGRAM=$(PROGRAM)

tool/extlibs.rb

@@ -0,0 +1,120 @@
+#!/usr/bin/ruby
+require 'fileutils'
+require 'digest'
+require_relative 'downloader'
+
+cache_dir = ".downloaded-cache"
+FileUtils.mkdir_p(cache_dir)
+
+def do_download(url, base, cache_dir)
+  Downloader.download(url, base, cache_dir, nil)
+end
+
+def do_checksum(cache, chksums)
+  chksums.each do |sum|
+    name, sum = sum.split(/:/)
+    if $VERBOSE
+      $stdout.print "checking #{name} of #{cache} ..."
+      $stdout.flush
+    end
+    hd = Digest(name.upcase).file(cache).hexdigest
+    if hd == sum
+      if $VERBOSE
+        $stdout.puts " OK"
+        $stdout.flush
+      end
+    else
+      if $VERBOSE
+        $stdout.puts " NG"
+        $stdout.flush
+      end
+      raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}"
+    end
+  end
+end
+
+def do_extract(cache, dir)
+  if $VERBOSE
+    $stdout.puts "extracting #{cache} into #{dir}"
+    $stdout.flush
+  end
+  Process.wait(Process.spawn("tar", "xpf", "-", in: cache, chdir: dir))
+  $?.success? or raise "failed to extract #{cache}"
+end
+
+def do_patch(dest, patch, args)
+  if $VERBOSE
+    $stdout.puts "applying #{patch} under #{dest}"
+    $stdout.flush
+  end
+  Process.wait(Process.spawn("patch", *args, in: File.join(dest, patch), chdir: dest))
+  $?.success? or raise "failed to patch #{patch}"
+end
+
+case ARGV[0]
+when '--download'
+  mode = :download
+  ARGV.shift
+when '--extract'
+  mode = :extract
+  ARGV.shift
+when '--patch'
+  mode = :patch
+  ARGV.shift
+when '--all'
+  mode = :all
+  ARGV.shift
+else
+  mode = :all
+end
+
+success = true
+ARGV.each do |dir|
+  Dir.glob("#{dir}/**/extlibs") do |list|
+    if $VERBOSE
+      $stdout.puts "downloading for #{list}"
+      $stdout.flush
+    end
+    extracted = false
+    dest = File.dirname(list)
+    IO.foreach(list) do |line|
+      line.sub!(/\s*#.*/, '')
+      if /^\t/ =~ line
+        if extracted and (mode == :all or mode == :patch)
+          patch, *args = line.split
+          do_patch(dest, patch, args)
+        end
+        next
+      end
+      url, *chksums = line.split(' ')
+      next unless url
+      extracted = false
+      base = File.basename(url)
+      cache = File.join(cache_dir, base)
+      target = File.join(dest, base[/.*(?=\.tar(?:\.\w+)?\z)/])
+      begin
+        case mode
+        when :download
+          do_download(url, base, cache_dir)
+          do_checksum(cache, chksums)
+        when :extract
+          unless File.directory?(target)
+            do_checksum(cache, chksums)
+            extracted = do_extract(cache, dest)
+          end
+        when :all
+          do_download(url, base, cache_dir)
+          unless File.directory?(target)
+            do_checksum(cache, chksums)
+            extracted = do_extract(cache, dest)
+          end
+        end
+      rescue => e
+        warn e.inspect
+        success = false
+      end
+    end
+  end
+end
+
+exit(success)

tool/make-snapshot

@@ -186,9 +186,15 @@ def package(vcs, rev, destdir, tmp = nil)
       return
     end
     if $srcdir
-      Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem}") do |file|
+      Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*}") do |file|
         puts "copying #{file}"
-        FileUtils.cp(file, exported + file[$srcdir.size..-1], preserve: true)
+        dest = exported + file[$srcdir.size..-1]
+        FileUtils.mkpath(File.dirname(dest))
+        begin
+          FileUtils.ln(file, dest, force: true)
+        rescue SystemCallError
+          FileUtils.cp(file, dest, preserve: true)
+        end
       end
     end
   end
@@ -280,6 +286,7 @@ def package(vcs, rev, destdir, tmp = nil)
                srcdir=. CHDIR=cd NULLCMD=:
                PATH_SEPARATOR=#{File::PATH_SEPARATOR}
                IFCHANGE=tool/ifchange MAKEDIRS=mkdir\ -p
+               RMALL=rm\ -fr
                MINIRUBY=#{miniruby}
                RUNRUBY=#{miniruby}
                RUBY=#{ENV["RUBY"]}
@@ -289,7 +296,7 @@ def package(vcs, rev, destdir, tmp = nil)
                prereq]
       IO.popen(cmd, "w") do |f|
         f.puts mk
-        f.puts "after-update::", "prereq: after-update"
+        f.puts "after-update::", "clean-cache $(CLEAN_CACHE): after-update", "prereq: clean-cache $(CLEAN_CACHE)"
       end
       clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk")
       print "prerequisites"