From 0af44e72611583963220e40215fc36f31fa80185 Mon Sep 17 00:00:00 2001 From: k0kubun Date: Sun, 4 Feb 2018 05:49:21 +0000 Subject: [PATCH] common.mk: install a single header file for JIT compilation which is created by transforming a preprocessed vm.c. This file will be used by JIT compiler's generated code which we are going to have from succeeding commits. Makefile.in: generate MJIT header for UNIX environments. win32/Makefile.sub: generate MJIT header for mswin environments. At initial merge, we're going to support only MinGW for Windows. So the header installed by this file won't be used for short term, but we'll add mswin support in a half year or so, for sure. tool/transform_mjit_header.rb: New. This script was originally written as minimize_mjit_header.rb by Vladimir N. Makarov for Feature 12589. Then I refactored a little so that it can conform CodeClimate CI which is currently set for Ruby's GitHub repository, and fixed some bugs and ported it to work on Windows. Also, as original minimize_mjit_header.rb takes too long time to run, this is modified to skip minimization step because having *static* unused definitions does not waste compilation time on -O2 since compiler can skip to compile unused static functions. So this does no longer "minimize" the header and is renamed. This header installation does NOT include a header to automatically export symbols used by MJIT. That's because original MJIT code was failing to export symbols in the import header in macOS environment. But I would like to have the functionality for maintainability in the future. I'll manually export things but it would be just an intemediate solution. Patch by: Vladimir N. Makarov Part of: Feature 12589 and 14235. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62187 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- .gitignore | 3 + Makefile.in | 10 ++ common.mk | 3 +- tool/transform_mjit_header.rb | 183 ++++++++++++++++++++++++++++++++++ win32/Makefile.sub | 9 ++ 5 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tool/transform_mjit_header.rb diff --git a/.gitignore b/.gitignore index efc7fd5864..3fa658820e 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,6 @@ lcov*.info # /win32/ /win32/*.ico /win32/.time + +# MJIT +/rb_mjit_header.h diff --git a/Makefile.in b/Makefile.in index 48dd90d32c..bca6696c9c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -54,6 +54,7 @@ DOCTARGETS = @RDOCTARGET@ @CAPITARGET@ EXTOUT = @EXTOUT@ arch_hdrdir = $(EXTOUT)/include/$(arch) VPATH = $(arch_hdrdir)/ruby:$(hdrdir)/ruby:$(srcdir):$(srcdir)/missing +MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h empty = CC_VERSION = @CC_VERSION@ @@ -407,6 +408,15 @@ probes.@OBJEXT@: $(srcdir)/probes.d $(DTRACE_REBUILD:yes=probes.stamp) $(Q) $(RM) $@ $(Q) $(DTRACE) -G -C $(INCFLAGS) -s $(srcdir)/probes.d -o $@ $(DTRACE_REBUILD_OBJS) +rb_mjit_header.h: PHONY probes.h + $(ECHO) building $@ + $(Q) $(CC) $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) -DMJIT_HEADER $(srcdir)/vm.c $(COUTFLAG) $@.new -E -P -dD + $(Q) (cmp $@.new $@ && $(ECHO0) $@ unchanged && $(RM) $@.new) || $(MV) $@.new $@ + +$(MJIT_MIN_HEADER): rb_mjit_header.h $(srcdir)/tool/transform_mjit_header.rb + $(ECHO) building $@ + $(BASERUBY) $(srcdir)/tool/transform_mjit_header.rb "$(CC)" rb_mjit_header.h $@ + # DTrace static library hacks described here: # http://mail.opensolaris.org/pipermail/dtrace-discuss/2005-August/000207.html ruby-glommed.$(OBJEXT): diff --git a/common.mk b/common.mk index 471ae838f7..28486baa03 100644 --- a/common.mk +++ b/common.mk @@ -59,6 +59,7 @@ ENC_TRANS_D = $(TIMESTAMPDIR)/.enc-trans.time RDOCOUT = $(EXTOUT)/rdoc HTMLOUT = $(EXTOUT)/html CAPIOUT = doc/capi +MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h INITOBJS = dmyext.$(OBJEXT) dmyenc.$(OBJEXT) NORMALMAINOBJ = main.$(OBJEXT) @@ -186,7 +187,7 @@ SHOWFLAGS = showflags all: $(SHOWFLAGS) main docs -main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs +main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs $(MJIT_MIN_HEADER) @$(NULLCMD) .PHONY: showflags diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb new file mode 100644 index 0000000000..308379123b --- /dev/null +++ b/tool/transform_mjit_header.rb @@ -0,0 +1,183 @@ +# Copyright (C) 2017 Vladimir Makarov, +# This is a script to transform functions to static inline. +# Usage: transform_mjit_header.rb
+ +require 'fileutils' +require 'tempfile' + +module MJITHeader + ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ + ATTR_REGEXP = /__attribute__\s*\(\((#{ATTR_VALUE_REGEXP})*\)\)/ + FUNC_HEADER_REGEXP = /\A(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ + + # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. + RECURSIVE_MACROS = %w[ + RASCTRYINFO + RASIPADDR + ] + + IGNORED_FUNCTIONS = [ + 'rb_equal_opt', # Not used from VM and not compilable + ] + + # Return start..stop of last decl in CODE ending STOP + def self.find_decl(code, stop) + level = 0 + stop.downto(0) do |i| + if level == 0 && stop != i && decl_found?(code, i) + return decl_start(code, i)..stop + elsif code[i] == '}' + level += 1 + elsif code[i] == '{' + level -= 1 + end + end + 0..-1 + end + + def self.decl_found?(code, i) + i == 0 || code[i] == ';' || code[i] == '}' + end + + def self.decl_start(code, i) + if i == 0 && code[i] != ';' && code[i] != '}' + 0 + else + i + 1 + end + end + + # Given DECL return the name of it, nil if failed + def self.decl_name_of(decl) + ident_regex = /\w+/ + decl = decl.gsub(/^#.+$/, '') # remove macros + reduced_decl = decl.gsub(/#{ATTR_REGEXP}/, '') # remove attributes + su1_regex = /{[^{}]*}/ + su2_regex = /{([^{}]|su1_regex)*}/ + su3_regex = /{([^{}]|su2_regex)*}/ # 3 nested structs/unions is probably enough + reduced_decl.gsub!(/#{su3_regex}/, '') # remove strutcs/unions in the header + id_seq_regex = /\s*(#{ident_regex}(\s+|\s*[*]+\s*))*/ + # Process function header: + match = /\A#{id_seq_regex}(?#{ident_regex})\s*\(/.match(reduced_decl) + return match[:name] if match + # Process non-function declaration: + reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization + match = /#{id_seq_regex}(?#{ident_regex})/.match(reduced_decl); + return match[:name] if match + nil + end + + # Return true if CC with CFLAGS compiles successfully the current code. + # Use STAGE in the message in case of a compilation failure + def self.check_code!(code, cc, cflags, stage) + Tempfile.open(['', '.c']) do |f| + f.puts code + f.close + unless system("#{cc} #{cflags} #{f.path} 2>#{File::NULL}") + STDERR.puts "error in #{stage} header file:" + system("#{cc} #{cflags} #{f.path}") + exit 1 + end + end + end + + # Remove unpreprocessable macros + def self.remove_harmful_macros!(code) + code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') + end + + # -dD outputs those macros, and it produces redefinition warnings + def self.remove_default_macros!(code) + code.gsub!(/^#define __STDC_.+$/, '') + code.gsub!(/^#define assert\([^\)]+\) .+$/, '') + end + + # This makes easier to process code + def self.separate_macro_and_code(code) + code.lines.partition { |l| !l.start_with?('#') }.flatten.join('') + end + + def self.write(code, out) + FileUtils.mkdir_p(File.dirname(out)) + File.write("#{out}.new", code) + FileUtils.mv("#{out}.new", out) + end + + # Note that this checks runruby. This conservatively covers platform names. + def self.windows? + RUBY_PLATFORM =~ /mswin|mingw|msys/ + end +end + +if ARGV.size != 3 + STDERR.puts 'Usage: transform_mjit_header.rb
' + exit 1 +end + +cc = ARGV[0] +code = File.read(ARGV[1]) # Current version of the header file. +outfile = ARGV[2] +if cc =~ /\Acl(\z| |\.exe)/ + cflags = '-DMJIT_HEADER -Zs' +else + cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' +end + +if MJITHeader.windows? + MJITHeader.remove_harmful_macros!(code) +end +MJITHeader.remove_default_macros!(code) + +# Check initial file correctness +MJITHeader.check_code!(code, cc, cflags, 'initial') + +if MJITHeader.windows? # transformation is broken with Windows headers for now + STDERR.puts "\nSkipped transforming external functions to static on Windows." + MJITHeader.write(code, outfile) + exit 0 +end +STDERR.puts "\nTransforming external functions to static:" + +code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW +stop_pos = code.match(/^#/).begin(0) # See `separate_macro_and_code`. This ignores proprocessors. +extern_names = [] + +# This loop changes function declarations to static inline. +loop do + decl_range = MJITHeader.find_decl(code, stop_pos) + break if decl_range.end < 0 + + stop_pos = decl_range.begin - 1 + decl = code[decl_range] + decl_name = MJITHeader.decl_name_of(decl) + + if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) + STDERR.puts "transform_mjit_header: changing definition of '#{decl_name}' to declaration" + code[decl_range] = decl.sub(/{.+}/m, ';') + elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) + decl.sub!(/(extern|static|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + STDERR.puts "transform_mjit_header: making declaration of '#{decl_name}' static inline" + end + + code[decl_range] = "static inline #{decl}" + elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ + extern_names << decl_name + decl[match.begin(0)...match.end(0)] = '' + + if decl =~ /static/ + STDERR.puts "warning: a static decl inside external definition of '#{decl_name}'" + end + + header.sub!(/(extern|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + STDERR.puts "transform_mjit_header: making external definition of '#{decl_name}' static inline" + end + code[decl_range] = "static inline #{header}#{decl}" + end +end + +# Check the final file correctness +MJITHeader.check_code!(code, cc, cflags, 'final') + +MJITHeader.write(code, outfile) diff --git a/win32/Makefile.sub b/win32/Makefile.sub index cf900fbe35..cbd36e0405 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -1186,6 +1186,15 @@ probes.h: {$(VPATH)}probes.dmyh #include "$(*F).dmyh" <