ruby--ruby/tool/update-deps

#!/usr/bin/ruby

# tool/update-deps verify makefile dependencies.

# Requirements:
#   gcc 4.5 (for -save-temps=obj option)
#   GNU make (for -p option)
#
# Warning: ccache (and similar tools) must be disabled for
# -save-temps=obj to work properly.
#
# Usage:
#   1. Compile ruby with -save-temps=obj option.
#      Ex.  ./configure debugflags='-save-temps=obj -g' && make all golf
#   2. Run tool/update-deps to show dependency problems.
#      Ex.  ruby tool/update-deps
#   3. Use --fix to fix makefiles.
#      Ex.  ruby tool/update-deps --fix
#
# Other usages:
#   *  Fix makefiles using previously detected dependency problems
#      Ex.  ruby tool/update-deps --actual-fix [file]
#      "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix".

require 'optparse'
require 'stringio'
require 'pathname'
require 'pp'

ENV['LC_ALL'] = 'C'

$opt_fix = false
$opt_a = false
$opt_actual_fix = false
$i_not_found = false

def optionparser
  op = OptionParser.new
  op.banner = 'Usage: ruby tool/update-deps'
  op.def_option('-a', 'show valid dependencies') { $opt_a = true }
  op.def_option('--fix') { $opt_fix = true }
  op.def_option('--actual-fix') { $opt_actual_fix = true }
  op
end

def read_make_deps(cwd)
  dependencies = {}
  make_p = `make -p all miniruby ruby golf 2> /dev/null`
  dirstack = [cwd]
  curdir = nil
  make_p.scan(%r{Entering\ directory\ ['`](.*)'|
                 ^\#\ (GNU\ Make)\ |
                 ^CURDIR\ :=\ (.*)|
                 ^([/0-9a-zA-Z._-]+):(.*)\n((?:\#.*\n)*)|
                 ^\#\ (Finished\ Make\ data\ base\ on)\ |
                 Leaving\ directory\ ['`](.*)'}x) {
    directory_enter = $1
    data_base_start = $2
    data_base_curdir = $3
    rule_target = $4
    rule_sources = $5
    rule_desc = $6
    data_base_end = $7
    directory_leave = $8
    #p $~
    if directory_enter
      enter_dir = Pathname(directory_enter)
      #p [:enter, enter_dir]
      dirstack.push enter_dir
    elsif data_base_start
      curdir = nil
    elsif data_base_curdir
      curdir = Pathname(data_base_curdir)
    elsif rule_target && rule_sources && rule_desc &&
          /Modification time never checked/ !~ rule_desc # This pattern match eliminates rules which VPATH is not expanded.
      target = rule_target
      deps = rule_sources
      deps = deps.scan(%r{[/0-9a-zA-Z._-]+})
      next if /\.o\z/ !~ target.to_s
      next if /\A\./ =~ target.to_s # skip rules such as ".c.o"
      #p [curdir, target, deps]
      dir = curdir || dirstack.last
      dependencies[dir + target] ||= []
      dependencies[dir + target] |= deps.map {|dep| dir + dep }
    elsif data_base_end
      curdir = nil
    elsif directory_leave
      leave_dir = Pathname(directory_leave)
      #p [:leave, leave_dir]
      if leave_dir != dirstack.last
        warn "unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}"
      end
      dirstack.pop
    end
  }
  dependencies
end

#def guess_compiler_wd(filename, hint0)
#  hint = hint0
#  begin
#    guess = hint + filename
#    if guess.file?
#      return hint
#    end
#    hint = hint.parent
#  end while hint.to_s != '.'
#  raise ArgumentError, "can not find #{filename} (hint: #{hint0})"
#end

def read_single_actual_deps(path_i, cwd)
  files = {}
  path_i.each_line.with_index {|line, lineindex|
    next if /\A\# \d+ "(.*)"/ !~ line
    files[$1] = lineindex
  }
  # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line.
  compiler_wd = files.keys.find {|f| %r{\A/.*//\z} =~ f }
  if compiler_wd
    files.delete compiler_wd
    compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, ''))
  else
    raise "compiler working directory not found"
  end
  deps = []
  files.each_key {|dep|
    next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc.
    dep = Pathname(dep)
    if dep.relative?
      dep = compiler_wd + dep
    end
    if !dep.file?
      warn "file not found: #{dep}"
      next
    end
    next if !dep.to_s.start_with?(cwd.to_s) # omit system headers.
    deps << dep
  }
  deps
end

def read_actual_deps(cwd)
  deps = {}
  Pathname.glob('**/*.o').sort.each {|fn_o|
    fn_i = fn_o.sub_ext('.i')
    if !fn_i.exist?
      warn "not found: #{fn_i}"
      $i_not_found = true
      next
    end
    path_o = cwd + fn_o
    path_i = cwd + fn_i
    deps[path_o] = read_single_actual_deps(path_i, cwd)
  }
  deps
end

def concentrate(dependencies, cwd)
  deps = {}
  dependencies.keys.sort.each {|target|
    sources = dependencies[target]
    target = target.relative_path_from(cwd)
    sources = sources.map {|s|
      rel = s.relative_path_from(cwd)
      rel
    }
    if %r{\A\.\.(/|\z)} =~ target.to_s
      warn "out of tree target: #{target}"
      next
    end
    sources = sources.reject {|s|
      if %r{\A\.\.(/|\z)} =~ s.to_s
        warn "out of tree source: #{s}"
        true
      else
        false
      end
    }
    deps[target] = sources
  }
  deps
end

def sort_paths(paths)
  paths.sort_by {|t|
    ary = t.to_s.split(%r{/})
    ary.map.with_index {|e, i| i == ary.length-1 ? [0, e] : [1, e] } # regular file first, directories last.
  }
end

def in_makefile(target, source)
  target = target.to_s
  source = source.to_s
  case target
  when %r{\A[^/]*\z}
    target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
    case source
    when 'newline.c', 'miniprelude.c', 'prelude.c' then source2 = source
    when 'thread_pthread.c' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).c'
    when 'thread_pthread.h' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).h'
    when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
    when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
    when 'revision.h' then source2 = '$(srcdir)/revision.h'
    when 'version.h' then source2 = '$(srcdir)/version.h'
    when 'include/ruby/version.h' then source2 = '$(srcdir)/include/ruby/version.h'
    when %r{\A[^/]*\z} then source2 = "{$(VPATH)}#{File.basename source}"
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Ainclude/ruby/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Aenc/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}"
    when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}"
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["common.mk", target2, source2]
  when %r{\Aenc/}
    target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
    case source
    when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
    when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $'
    when %r{\Ainclude/ruby/} then source2 = $'
    when %r{\Aenc/} then source2 = source
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["enc/depend", target2, source2]
  when %r{\Aext/}
    unless File.exist?("#{File.dirname(target)}/extconf.rb")
      warn "not found: #{File.dirname(target)}/extconf.rb"
    end
    target2 = File.basename(target)
    case source
    when 'include/ruby.h' then source2 = '$(top_srcdir)/include/ruby.h'
    when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}"
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}"
    when %r{\A#{Regexp.escape File.dirname(target)}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)"
    when %r{\A#{Regexp.escape File.dirname(target)}/} then source2 = $'
    when 'id.h' then source2 = '{$(VPATH)}id.h'
    when 'parse.h' then source2 = '{$(VPATH)}parse.h'
    when 'lex.c' then source2 = '{$(VPATH)}lex.c'
    when 'probes.h' then source2 = '{$(VPATH)}probes.h'
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["#{File.dirname(target)}/depend", target2, source2]
  else
    raise "unexpected target: #{target}"
  end
end

def compare_deps(make_deps, actual_deps, out=$stdout)
  targets = sort_paths(actual_deps.keys)
  targets.each {|target|
    actual_sources = actual_deps[target]
    if !make_deps.has_key?(target)
      warn "no makefile dependency for #{target}"
    else
      make_sources = make_deps[target]
      sort_paths(actual_sources | make_sources).each {|source|
        makefile, target2, source2 = in_makefile(target, source)
        lines = begin
          File.readlines(makefile)
        rescue Errno::ENOENT
          []
        end
        #depline = "#{target2}: #{source2} \# #{target}: #{source}\n"
        depline = "#{target2}: #{source2}\n"
        if !make_sources.include?(source)
          out.puts "add #{makefile} : #{depline}"
        elsif !actual_sources.include?(source)
          if lines.include? depline
            out.puts "delL #{makefile} : #{depline}" # delL stands for del line
          else
            out.puts "delP #{makefile} : #{depline}" # delP stands for del prerequisite
          end
        else
          if $opt_a
            if lines.include? depline
              out.puts "okL #{makefile} : #{depline}" # okL stands for ok line
            else
              out.puts "okP #{makefile} : #{depline}" # okP stands for ok prerequisite
            end
          end
        end
      }
    end
  }
end

def main_show(out=$stdout)
  cwd = Pathname.pwd
  make_deps = read_make_deps(cwd)
  #pp make_deps
  make_deps = concentrate(make_deps, cwd)
  #pp make_deps
  actual_deps = read_actual_deps(cwd)
  #pp actual_deps
  actual_deps = concentrate(actual_deps, cwd)
  #pp actual_deps
  compare_deps(make_deps, actual_deps, out)
end

def extract_deplines(problems)
  adds = {}
  dels = {}
  problems.each_line {|line|
    case line
    when /\Aadd (\S+) : (\S.*\n)\z/
      (adds[$1] ||= []) << $2
    when /\AdelL (\S+) : (\S.*\n)\z/
      (dels[$1] ||= []) << $2
    when /\AdelP (\S+) : (\S.*\n)\z/
      (dels[$1] ||= []) << $2
    when /\AokL (\S+) : (\S.*\n)\z/
    when /\AokP (\S+) : (\S.*\n)\z/
      (adds[$1] ||= []) << $2
    end
  }
  return adds, dels
end

DEPENDENCIES_SECTION_START_MARK = "\# AUTOGENERATED DEPENDENCIES START\n"
DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n"

def main_actual_fix(problems)
  adds, dels = extract_deplines(problems)
  (adds.keys | dels.keys).sort.each {|makefile|
    content = begin
      File.read(makefile)
    rescue Errno::ENOENT
      ''
    end
    if /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK}((?:.*\n)*)#{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/ =~ content
      pre_post_part = [$`, $']
      lines = $1.lines.to_a
    else
      pre_post_part = nil
      lines = []
    end

    lines_original = lines.dup

    if dels[makefile]
      lines -= dels[makefile]
    end
    if adds[makefile]
      lines.concat(adds[makefile] - lines)
    end

    if lines == lines_original
      next
    end

    if pre_post_part
      new_content = [
        pre_post_part.first,
        DEPENDENCIES_SECTION_START_MARK,
        *lines,
        DEPENDENCIES_SECTION_END_MARK,
        pre_post_part.last
      ].join
      tmp_makefile = "#{makefile}.new#{$$}"
      File.write(tmp_makefile, new_content)
      File.rename tmp_makefile, makefile
      puts "modified: #{makefile}"
    else
      new_content = [
        DEPENDENCIES_SECTION_START_MARK,
        *lines,
        DEPENDENCIES_SECTION_END_MARK,
      ].join
      if !File.exist?(makefile)
        if !lines.empty?
          File.open(makefile, 'w') {|f|
            f.print new_content
          }
          puts "created: #{makefile}"
        end
      else
        puts "no dependencies section: #{makefile}"
        (lines_original - lines).each {|line|
          puts "  del: #{line}"
        }
        (lines - lines_original).each {|line|
          puts "  add: #{line}"
        }
      end
    end
  }
end

def main_fix
  problems = StringIO.new
  main_show(problems)
  main_actual_fix(problems.string)
end

def run
  op = optionparser
  op.parse!(ARGV)
  if $opt_actual_fix
    main_actual_fix(ARGF.read)
  elsif $opt_fix
    main_fix
  else
    main_show
  end
end

run
if $i_not_found
  warn "missing *.i files, see help in #$0 and ensure ccache is disabled"
end