id.h: independent from parse.h

* template/id.h.tmpl, tool/id2token.rb: make id.h independent from
  parse.h, and make parse.c dependent on it instead.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@36864 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,8 @@
+Fri Aug 31 14:31:17 2012  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* template/id.h.tmpl, tool/id2token.rb: make id.h independent from
+	  parse.h, and make parse.c dependent on it instead.
+
 Fri Aug 31 14:27:39 2012  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* lib/mkmf.rb (create_makefile): fix race conditions at install-ext.

common.mk

@@ -27,7 +27,6 @@ REVISION_H    = ./.revision.time
 PLATFORM_D    = ./$(PLATFORM_DIR)/.time
 RDOCOUT       = $(EXTOUT)/rdoc
 CAPIOUT       = doc/capi
-ID_H_TARGET   = ./.id.h.time
 
 DMYEXT	      = dmyext.$(OBJEXT)
 NORMALMAINOBJ = main.$(OBJEXT)
@@ -105,7 +104,7 @@ EXPORTOBJS    = dln.$(OBJEXT) \
 		$(COMMONOBJS)
 
 OBJS          = $(EXPORTOBJS) prelude.$(OBJEXT)
-ALLOBJS       = $(ID_H_TARGET) $(NORMALMAINOBJ) $(MINIOBJS) $(COMMONOBJS) $(DMYEXT)
+ALLOBJS       = $(NORMALMAINOBJ) $(MINIOBJS) $(COMMONOBJS) $(DMYEXT)
 
 GOLFOBJS      = goruby.$(OBJEXT) golf_prelude.$(OBJEXT)
 
@@ -561,20 +560,17 @@ $(ENC_MK): $(srcdir)/enc/make_encmake.rb $(srcdir)/enc/Makefile.in $(srcdir)/enc
 
 PHONY:
 
-{$(VPATH)}parse.c: {$(VPATH)}parse.y $(srcdir)/tool/ytab.sed
+{$(VPATH)}parse.c: {$(VPATH)}parse.y $(srcdir)/tool/ytab.sed {$(VPATH)}id.h
 {$(VPATH)}parse.h: {$(VPATH)}parse.c
 
 {$(srcdir)}.y.c:
 	$(ECHO) generating $@
-	$(Q)$(YACC) -d $(YFLAGS) -o y.tab.c $(SRC_FILE)
+	$(Q)$(BASERUBY) $(srcdir)/tool/id2token.rb --vpath=$(VPATH) id.h $(SRC_FILE) > parse.tmp.y
-	$(Q)sed -f $(srcdir)/tool/ytab.sed -e "/^#/s!y\.tab\.c!$@!" y.tab.c > $@.new
+	$(Q)$(YACC) -d $(YFLAGS) -o y.tab.c parse.tmp.y
+	$(Q)$(RM) parse.tmp.y
+	$(Q)sed -f $(srcdir)/tool/ytab.sed -e "/^#/s!parse\.tmp\.[iy]!parse.y!" -e "/^#/s!y\.tab\.c!$@!" y.tab.c > $@.new
 	$(Q)$(MV) $@.new $@
-	$(Q)$(RM) y.tab.c y.tab.h
+	$(Q)sed -e "/^#line.*y\.tab\.h/d;/^#line.*parse.*\.y/d" y.tab.h > $(@:.c=.h)
-
-{$(srcdir)}.y.h:
-	$(ECHO) generating $@
-	$(Q)$(YACC) -d $(YFLAGS) -o y.tab.c $(SRC_FILE)
-	$(Q)sed -e "/^#line.*y\.tab\.h/d;/^#line.*parse\.y/d" y.tab.h > $(@:.c=.h)
 	$(Q)$(RM) y.tab.c y.tab.h
 
 acosh.$(OBJEXT): {$(VPATH)}acosh.c
@@ -847,15 +843,14 @@ srcs-enc: $(ENC_MK)
 	$(Q) $(MAKE) -f $(ENC_MK) RUBY="$(MINIRUBY)" MINIRUBY="$(MINIRUBY)" $(MFLAGS) srcs
 
 incs: $(INSNS) {$(VPATH)}node_name.inc {$(VPATH)}encdb.h {$(VPATH)}transdb.h {$(VPATH)}known_errors.inc \
-      $(srcdir)/revision.h $(REVISION_H) enc/unicode/name2ctype.h $(ID_H_TARGET)
+      $(srcdir)/revision.h $(REVISION_H) enc/unicode/name2ctype.h {$(VPATH)}id.h
 
 insns: $(INSNS)
 
-$(ID_H_INCLUDES) $(ID_H_TARGET): {$(VPATH)}parse.h $(srcdir)/tool/generic_erb.rb $(srcdir)/template/id.h.tmpl
+{$(VPATH)}id.h: $(srcdir)/tool/generic_erb.rb $(srcdir)/template/id.h.tmpl
-	$(ECHO) generating id.h
+	$(ECHO) generating $@
-	$(Q) $(BASERUBY) $(srcdir)/tool/generic_erb.rb --if-change --output=id.h \
+	$(Q) $(BASERUBY) $(srcdir)/tool/generic_erb.rb --output=$@ \
-		--vpath=$(VPATH) --timestamp \
+		$(srcdir)/template/$@.tmpl
-		$(srcdir)/template/id.h.tmpl parse.h
 
 node_name.inc: {$(VPATH)}node.h
 	$(ECHO) generating $@
@@ -904,7 +899,7 @@ $(REVISION_H): $(srcdir)/version.h $(srcdir)/ChangeLog $(srcdir)/tool/file2lastr
 $(srcdir)/ext/ripper/ripper.c: parse.y
 	$(ECHO) generating $@
 	$(Q) $(CHDIR) $(@D) && $(exec) $(MAKE) -f depend $(MFLAGS) \
-		Q=$(Q) ECHO=$(ECHO) top_srcdir=../.. srcdir=. RUBY="$(BASERUBY)"
+		Q=$(Q) ECHO=$(ECHO) top_srcdir=../.. srcdir=. VPATH=../.. RUBY="$(BASERUBY)"
 
 $(srcdir)/ext/json/parser/parser.c: $(srcdir)/ext/json/parser/parser.rl
 	$(ECHO) generating $@

ext/ripper/depend

@@ -25,7 +25,9 @@ static: check
 
 ripper.y: $(srcdir)/tools/preproc.rb $(top_srcdir)/parse.y
 	$(ECHO) extracting $@ from $(top_srcdir)/parse.y
-	$(Q) $(RUBY) $(srcdir)/tools/preproc.rb $(top_srcdir)/parse.y --output=$@
+	$(Q) $(RUBY) $(top_srcdir)/tool/id2token.rb --vpath=$(VPATH) id.h $(top_srcdir)/parse.y > ripper.tmp.y
+	$(Q) $(RUBY) $(srcdir)/tools/preproc.rb ripper.tmp.y --output=$@
+	$(Q) $(RM) ripper.tmp.y
 
 check: $(GEN) $(SRC1) $(SRC2)
 	$(ECHO) checking $(SRC1) and $(SRC2)

parse.y

@@ -744,25 +744,25 @@ static void token_info_pop(struct parser_params*, const char *token);
 %type <val> program reswords then do dot_or_colon
 %*/
 %token END_OF_INPUT 0	"end-of-input"
-%token tUPLUS		"unary+"
+%token tUPLUS		RUBY_TOKEN(UPLUS)  "unary+"
-%token tUMINUS		"unary-"
+%token tUMINUS		RUBY_TOKEN(UMINUS) "unary-"
-%token tPOW		"**"
+%token tPOW		RUBY_TOKEN(POW)    "**"
-%token tCMP		"<=>"
+%token tCMP		RUBY_TOKEN(CMP)    "<=>"
-%token tEQ		"=="
+%token tEQ		RUBY_TOKEN(EQ)     "=="
-%token tEQQ		"==="
+%token tEQQ		RUBY_TOKEN(EQQ)    "==="
-%token tNEQ		"!="
+%token tNEQ		RUBY_TOKEN(NEQ)    "!="
-%token tGEQ		">="
+%token tGEQ		RUBY_TOKEN(GEQ)    ">="
-%token tLEQ		"<="
+%token tLEQ		RUBY_TOKEN(LEQ)    "<="
 %token tANDOP		"&&"
 %token tOROP		"||"
-%token tMATCH		"=~"
+%token tMATCH		RUBY_TOKEN(MATCH)  "=~"
-%token tNMATCH		"!~"
+%token tNMATCH		RUBY_TOKEN(NMATCH) "!~"
-%token tDOT2		".."
+%token tDOT2		RUBY_TOKEN(DOT2)   ".."
-%token tDOT3		"..."
+%token tDOT3		RUBY_TOKEN(DOT3)   "..."
-%token tAREF		"[]"
+%token tAREF		RUBY_TOKEN(AREF)   "[]"
-%token tASET		"[]="
+%token tASET		RUBY_TOKEN(ASET)   "[]="
-%token tLSHFT		"<<"
+%token tLSHFT		RUBY_TOKEN(LSHFT)  "<<"
-%token tRSHFT		">>"
+%token tRSHFT		RUBY_TOKEN(RSHFT)  ">>"
 %token tCOLON2		"::"
 %token tCOLON3		":: at EXPR_BEG"
 %token <id> tOP_ASGN	/* +=, -=  etc. */
@@ -9918,8 +9918,8 @@ static struct symbols {
     st_table *ivar2_id;
     st_table *id_ivar2;
 #endif
-    VALUE op_sym[tLAST_TOKEN];
+    VALUE op_sym[tLAST_OP_ID];
-} global_symbols = {tLAST_ID};
+} global_symbols = {tLAST_TOKEN};
 
 static const struct st_hash_type symhash = {
     rb_str_hash_cmp,
@@ -9977,7 +9977,7 @@ rb_gc_mark_symbols(void)
 {
     rb_mark_tbl(global_symbols.id_str);
     rb_gc_mark_locations(global_symbols.op_sym,
-			 global_symbols.op_sym + tLAST_TOKEN);
+			 global_symbols.op_sym + numberof(global_symbols.op_sym));
 }
 #endif /* !RIPPER */
 
@@ -10264,7 +10264,7 @@ intern_str(VALUE str)
 	if (m[last] == '=') {
 	    /* attribute assignment */
 	    id = rb_intern3(name, last, enc);
-	    if (id > tLAST_TOKEN && !is_attrset_id(id)) {
+	    if (id > tLAST_OP_ID && !is_attrset_id(id)) {
 		enc = rb_enc_get(rb_id2str(id));
 		id = rb_id_attrset(id);
 		goto id_register;

template/id.h.tmpl

@@ -13,14 +13,7 @@
 <%
 require 'optparse'
 
-input = ARGV.shift or abort opt.opt_s
+op_id_offset = 128
-
-tokens = nil
-vpath.open(input) do |f|
-  if line = f.read[/^\s*enum\s+yytokentype\s*\{([^{}]*)\s*\};/m, 1]
-    tokens = line.scan(/\b(t(?:LAST_TOKEN|U(?:PLUS|MINUS)|POW|CMP|EQQ?|[NGL]EQ|(?:AND|OR)OP|N?MATCH|DOT\d|AREF|ASET|[LR]SHFT|LAMBDA)|id\w+)\s*=\s*(\d+),?/m)
-  end
-end
 
 method_ids = %w[
   Intern
@@ -39,6 +32,12 @@ method_ids = %w[
   Answer
 ]
 
+token_op_ids = %w[
+  tDOT2 tDOT3 tUPLUS tUMINUS tPOW tDSTAR tCMP tLSHFT tRSHFT
+  tLEQ tGEQ tEQ tEQQ tNEQ tMATCH tNMATCH tAREF tASET
+  tCOLON2 tCOLON3
+]
+
 preserved_ids = %w[
   NULL
   Respond_to
@@ -70,74 +69,54 @@ preserved_ids = %w[
 #define ID_JUNK       0x07
 #define ID_INTERNAL   ID_JUNK
 
-#ifdef USE_PARSE_H
-#include "parse.h"
-#endif
-
 #define symIFUNC ID2SYM(idIFUNC)
 #define symCFUNC ID2SYM(idCFUNC)
 
-#if !defined tLAST_TOKEN && defined YYTOKENTYPE
+% token_op_ids.each_with_index do |token, index|
-#define tLAST_TOKEN tLAST_TOKEN
+#define RUBY_TOKEN_<%=token[/\At(.+)\z/, 1]%> <%=op_id_offset + index%>
-#endif
+% end
+#define RUBY_TOKEN(t) RUBY_TOKEN_##t
 
 enum ruby_method_ids {
-#ifndef tLAST_TOKEN
+    idDot2 = RUBY_TOKEN(DOT2),
-% tokens.each do |token, value|
+    idDot3 = RUBY_TOKEN(DOT3),
-    <%=token%> = <%=value%>,
+    idUPlus = RUBY_TOKEN(UPLUS),
-% end
+    idUMinus = RUBY_TOKEN(UMINUS),
-#endif
+    idPow = RUBY_TOKEN(POW),
-    idDot2 = tDOT2,
+    idCmp = RUBY_TOKEN(CMP),
-    idDot3 = tDOT3,
-    idUPlus = tUPLUS,
-    idUMinus = tUMINUS,
-    idPow = tPOW,
-    idCmp = tCMP,
     idPLUS = '+',
     idMINUS = '-',
     idMULT = '*',
     idDIV = '/',
     idMOD = '%',
     idLT = '<',
-    idLTLT = tLSHFT,
+    idLTLT = RUBY_TOKEN(LSHFT),
-    idLE = tLEQ,
+    idLE = RUBY_TOKEN(LEQ),
     idGT = '>',
-    idGE = tGEQ,
+    idGE = RUBY_TOKEN(GEQ),
-    idEq = tEQ,
+    idEq = RUBY_TOKEN(EQ),
-    idEqq = tEQQ,
+    idEqq = RUBY_TOKEN(EQQ),
-    idNeq = tNEQ,
+    idNeq = RUBY_TOKEN(NEQ),
     idNot = '!',
     idBackquote = '`',
-    idEqTilde = tMATCH,
+    idEqTilde = RUBY_TOKEN(MATCH),
-    idNeqTilde = tNMATCH,
+    idNeqTilde = RUBY_TOKEN(NMATCH),
-    idAREF = tAREF,
+    idAREF = RUBY_TOKEN(AREF),
-    idASET = tASET,
+    idASET = RUBY_TOKEN(ASET),
-    tPRESERVED_ID_BEGIN = tLAST_TOKEN-1,
+    tPRESERVED_ID_BEGIN = <%=op_id_offset + token_op_ids.size - 1%>,
 % preserved_ids.each do |token|
     id<%=token%>,
 % end
     tPRESERVED_ID_END,
-    tLAST_OP_ID = tPRESERVED_ID_END-1,
-    idLAST_OP_ID = tLAST_OP_ID >> ID_SCOPE_SHIFT,
 % method_ids.each do |token|
     t<%=token%>,
 % end
-    tLAST_ID,
 #define TOKEN2ID(n) id##n = ((t##n<<ID_SCOPE_SHIFT)|ID_LOCAL)
 % method_ids.each do |token|
     TOKEN2ID(<%=token%>),
 % end
-    TOKEN2ID(LAST_ID)
+    tLAST_OP_ID = tPRESERVED_ID_END-1,
+    idLAST_OP_ID = tLAST_OP_ID >> ID_SCOPE_SHIFT
 };
 
-#ifdef tLAST_TOKEN
-struct ruby_method_ids_check {
-#define ruby_method_id_check_for(name, value) \
-    int checking_for_##name[name == (value) ? 1 : -1]
-% tokens.map do |token, value|
-ruby_method_id_check_for(<%=token%>, <%=value%>);
-% end
-};
-#endif
-
 #endif /* RUBY_ID_H */

tool/id2token.rb

@@ -0,0 +1,29 @@
+#! /usr/bin/ruby -p
+BEGIN {
+  require 'optparse'
+  vpath = ["."]
+  header = nil
+
+  opt = OptionParser.new do |o|
+    o.on('-v', '--vpath=DIR') {|dirs| vpath.concat dirs.split(File::PATH_SEPARATOR)}
+    header = o.order!(ARGV).shift
+  end or abort opt.opt_s
+
+  TOKENS = {}
+  vpath.find do |dir|
+    begin
+      h = File.read(File.join(dir, header))
+    rescue Errno::ENOENT
+      nil
+    else
+      h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id|
+        TOKENS[token] = id
+      end
+      true
+    end
+  end or abort "#{header} not found in #{vpath.inspect}"
+
+  TOKENS_RE = /\bRUBY_TOKEN\((#{TOKENS.keys.join('|')})\)\s*(?=\s)/
+}
+
+$_.gsub!(TOKENS_RE) {TOKENS[$1]} if /^%token/ =~ $_