* parse.y [ripper]: adjust lineno and columns for multi-line strings.

* parse.y [ripper]: delay heredocument events until seeing end-of-line.
* parse.y [ripper]: event on__heredoc_contentn -> on__tstring_content.
* ext/ripper/eventids2.c: ditto.
* ext/ripper/lib/ripper.rb: sync with eventids2.c.
* test/ripper/test_scanner_events.rb: test it.
* ext/ripper/tools/generate-ripper_rb.rb: show basename of input.
* ext/ripper/Makefile.dev: support objdir build.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6931 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,24 @@
+Mon Sep 20 14:39:42 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* parse.y [ripper]: adjust lineno and columns for multi-line
+	  strings.
+
+	* parse.y [ripper]: delay heredocument events until seeing
+	  end-of-line.
+
+	* parse.y [ripper]: event on__heredoc_contentn ->
+	  on__tstring_content.
+
+	* ext/ripper/eventids2.c: ditto.
+
+	* ext/ripper/lib/ripper.rb: sync with eventids2.c.
+
+	* test/ripper/test_scanner_events.rb: test it.
+
+	* ext/ripper/tools/generate-ripper_rb.rb: show basename of input.
+
+	* ext/ripper/Makefile.dev: support objdir build.
+
 Mon Sep 20 13:22:55 2004  Minero Aoki  <aamine@loveruby.net>
 
 	* parse.y [ripper]: remove Ripper#pos.

ext/ripper/Makefile.dev

@@ -15,4 +15,4 @@ test:
 
 preproc: ripper.E
 ripper.E: ripper.c
-	$(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@
+	$(CC) -E $(CPPFLAGS) ripper.c | ruby $(srcdir)/tools/strip.rb > $@

ext/ripper/eventids2.c

@@ -5,10 +5,9 @@
 #define tEMBDOC_END      (tLAST_TOKEN + 5)
 #define tSP              (tLAST_TOKEN + 6)
 #define tHEREDOC_BEG     (tLAST_TOKEN + 7)
-#define tHEREDOC_CONTENT (tLAST_TOKEN + 8)
+#define tHEREDOC_END     (tLAST_TOKEN + 8)
-#define tHEREDOC_END     (tLAST_TOKEN + 9)
+#define k__END__         (tLAST_TOKEN + 9)
-#define k__END__         (tLAST_TOKEN + 10)
+#define tCHAR            (tLAST_TOKEN + 10)
-#define tCHAR            (tLAST_TOKEN + 11)
 
 static ID ripper_id_scan;
 
@@ -53,7 +52,6 @@ static ID ripper_id_embdoc;
 static ID ripper_id_embdoc_end;
 static ID ripper_id_sp;
 static ID ripper_id_heredoc_beg;
-static ID ripper_id_heredoc_content;
 static ID ripper_id_heredoc_end;
 static ID ripper_id___end__;
 static ID ripper_id_CHAR;
@@ -104,7 +102,6 @@ ripper_init_eventids2()
     ripper_id_embdoc_end = rb_intern("on__embdoc_end");
     ripper_id_sp = rb_intern("on__sp");
     ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
-    ripper_id_heredoc_content = rb_intern("on__heredoc_content");
     ripper_id_heredoc_end = rb_intern("on__heredoc_end");
     ripper_id___end__ = rb_intern("on____end__");
     ripper_id_CHAR = rb_intern("on__CHAR");
@@ -250,7 +247,6 @@ static struct token_assoc {
     {tEMBDOC_END,       &ripper_id_embdoc_end},
     {tSP,               &ripper_id_sp},
     {tHEREDOC_BEG,      &ripper_id_heredoc_beg},
-    {tHEREDOC_CONTENT,  &ripper_id_heredoc_content},
     {tHEREDOC_END,      &ripper_id_heredoc_end},
     {k__END__,          &ripper_id___end__},
     {0, NULL}

ext/ripper/lib/ripper.rb

@@ -1,5 +1,5 @@
 #
-# This file is automatically generated from /home/aamine/c/ruby/ext/ripper/ripper.rb.in and parse.y.
+# This file is automatically generated from ripper.rb.in and parse.y.
 # DO NOT MODIFY!!!!!!
 #
 #
@@ -165,7 +165,6 @@ class Ripper
     :float => 1,
     :gvar => 1,
     :heredoc_beg => 1,
-    :heredoc_content => 1,
     :heredoc_end => 1,
     :ident => 1,
     :ignored_nl => 1,
@@ -774,10 +773,6 @@ class Ripper
     token
   end
 
-  def on__heredoc_content(token)
-    token
-  end
-
   def on__heredoc_end(token)
     token
   end

ext/ripper/tools/generate-ripper_rb.rb

@@ -4,7 +4,7 @@ def main
   template, ids1, ids2 = *ARGV
   print <<header
 #
-# This file is automatically generated from #{template} and parse.y.
+# This file is automatically generated from #{File.basename(template)} and parse.y.
 # DO NOT MODIFY!!!!!!
 #
 header

parse.y

@@ -144,8 +144,10 @@ struct parser_params {
     int parser_ruby__end__seen;
     int parser_ruby_sourceline;
     VALUE parser_ruby_sourcefile;
-    VALUE delayed;
     char *tokp;
+    int current_t;
+    int in_heredoc;
+    VALUE delayed;
 #endif
 };
 
@@ -4148,7 +4150,9 @@ none		: /* none */
 # undef yylval
 # define yylval  (*((YYSTYPE*)(parser->parser_yylval)))
 
+#ifndef RIPPER
 static struct parser_params* parser_new _((void));
+#endif
 static int parser_regx_options _((struct parser_params*));
 static int parser_tokadd_string _((struct parser_params*,int,int,int,long*));
 static int parser_parse_string _((struct parser_params*,NODE*));
@@ -4188,31 +4192,6 @@ static int parser_here_document _((struct parser_params*,NODE*));
 #endif
 
 #ifdef RIPPER
-static void
-ripper_save_token(parser)
-    struct parser_params *parser;
-{
-    rb_ary_push(parser->delayed,
-		rb_str_new(parser->tokp, lex_pend - parser->tokp));
-}
-
-static void
-ripper_dispatch_delayed(parser, t)
-    struct parser_params *parser;
-    int t;
-{
-    long i;
-
-    for (i = 0; i < RARRAY(parser->delayed)->len; i++) {
-        ID event = ripper_token2eventid(t);
-
-        ripper_dispatch2(parser, ripper_id_scan,
-                         ID2SYM(event), RARRAY(parser->delayed)->ptr[i]);
-        ripper_dispatch1(parser, event, RARRAY(parser->delayed)->ptr[i]);
-    }
-    rb_ary_clear(parser->delayed);
-}
-
 #define ripper_flush(p) (p->tokp = p->parser_lex_p)
 
 static void
@@ -4220,14 +4199,55 @@ ripper_dispatch_scan_event(parser, t)
     struct parser_params *parser;
     int t;
 {
-    if (lex_p > parser->tokp) {
+    VALUE str;
-        VALUE str = rb_str_new(parser->tokp, lex_p - parser->tokp);
+    ID event;
-        ID event = ripper_token2eventid(t);
 
-        ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
+    if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
-        ripper_dispatch1(parser, event, str);
+    if (lex_p == parser->tokp) return;
-        ripper_flush(parser);
+    str = rb_str_new(parser->tokp, lex_p - parser->tokp);
+    event = ripper_token2eventid(t);
+    if (parser->in_heredoc) {
+	int col = parser->tokp - lex_pbeg;
+	rb_ary_push(parser->delayed, ID2SYM(event));
+	rb_ary_push(parser->delayed, str);
+	rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline));
+	rb_ary_push(parser->delayed, INT2NUM(col));
+	ripper_flush(parser);
+	return;
     }
+    ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
+    ripper_dispatch1(parser, event, str);
+    ripper_flush(parser);
+}
+
+static void
+ripper_dispatch_delayed_events(parser)
+    struct parser_params *parser;
+{
+    long i;
+    int saved_line;
+    char *saved_tokp;
+
+    if (RARRAY(parser->delayed)->len == 0) return;
+    if (RARRAY(parser->delayed)->len % 4 != 0)
+    	rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0");
+    saved_line = ruby_sourceline;
+    saved_tokp = parser->tokp;
+    i = 0;
+    while (i < RARRAY(parser->delayed)->len) {
+	VALUE event, str, vline, vcol;
+	event = RARRAY(parser->delayed)->ptr[i++];
+	str = RARRAY(parser->delayed)->ptr[i++];
+	vline = RARRAY(parser->delayed)->ptr[i++];
+	vcol = RARRAY(parser->delayed)->ptr[i++];
+	ruby_sourceline = NUM2INT(vline);
+	parser->tokp = lex_pbeg + NUM2INT(vcol);
+	ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str));
+	ripper_dispatch1(parser, SYM2ID(event), str);
+    }
+    rb_ary_clear(parser->delayed);
+    ruby_sourceline = saved_line;
+    parser->tokp = saved_tokp;
 }
 #endif /* RIPPER */
 
@@ -4465,9 +4485,11 @@ parser_nextc(parser)
                 return -1;
             }
 #ifdef RIPPER
-            if (parser->tokp < lex_pend) {
+	    if (parser->tokp < lex_pend) {
-                ripper_save_token(parser);
+		if (!parser->current_t)
-            }
+		    rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t");
+		ripper_dispatch_scan_event(parser, parser->current_t);
+	    }
 #endif
 	    if (heredoc_end > 0) {
 		ruby_sourceline = heredoc_end;
@@ -4917,6 +4939,9 @@ parser_parse_string(parser, quote)
     if (func == -1) return tSTRING_END;
     c = nextc();
     if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
+#ifdef RIPPER
+	parser->current_t = ' ';
+#endif
 	do {c = nextc();} while (ISSPACE(c));
 	space = 1;
     }
@@ -5032,7 +5057,6 @@ parser_heredoc_restore(parser, here)
     VALUE line;
 
 #ifdef RIPPER
-    ripper_dispatch_delayed(parser, tHEREDOC_CONTENT);
     lex_goto_eol(parser);
     ripper_dispatch_scan_event(parser, tHEREDOC_END);
 #endif
@@ -5047,6 +5071,7 @@ parser_heredoc_restore(parser, here)
     rb_gc_force_recycle((VALUE)here);
 #ifdef RIPPER
     ripper_flush(parser);
+    parser->in_heredoc = Qfalse;
 #endif
 }
 
@@ -5197,7 +5222,13 @@ parser_yylex(parser)
 
     if (lex_strterm) {
 	int token;
+#ifdef RIPPER
+	parser->current_t = tSTRING_CONTENT;
+#endif
 	if (nd_type(lex_strterm) == NODE_HEREDOC) {
+#ifdef RIPPER
+	    parser->in_heredoc = Qtrue;
+#endif
 	    token = here_document(lex_strterm);
 	    if (token == tSTRING_END) {
 		lex_strterm = 0;
@@ -6413,9 +6444,11 @@ yylex(p)
 #endif
     t = parser_yylex(parser);
 #ifdef RIPPER
+    parser->current_t = 0;
     if (t != 0) {
-	ripper_dispatch_delayed(parser, t);
 	ripper_dispatch_scan_event(parser, t);
+	if (t == '\n' && !parser->in_heredoc)
+	    ripper_dispatch_delayed_events(parser);
     }
 #endif
 
@@ -8105,6 +8138,8 @@ parser_initialize(parser)
     parser->parser_lex_p = 0;
     parser->parser_lex_pend = 0;
 #ifdef RIPPER
+    parser->current_t = 0;
+    parser->in_heredoc = Qfalse;
     parser->delayed = rb_ary_new();
 #endif
 }
@@ -8139,6 +8174,7 @@ ripper_free(ptr)
     free(p);
 }
 
+#ifndef RIPPER
 struct parser_params *
 parser_new()
 {
@@ -8150,9 +8186,9 @@ parser_new()
     parser_initialize(p);
     return p;
 }
+#endif
 
 #ifdef RIPPER
-
 #ifdef RIPPER_DEBUG
 extern int rb_is_pointer_to_heap _((VALUE));
 

test/ripper/test_scanner_events.rb

@@ -573,19 +573,21 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS")
     assert_equal [%q(<<`EOS`)],
                  R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS")
+    assert_equal [%q(<<" ")],
+                 R.scan('heredoc_beg', %Q[<<" "\nheredoc\nEOS])
   end
 
-  def test_heredoc_content
+  def test_tstring_content_HEREDOC
     assert_equal [],
-                 R.scan('heredoc_content', '')
+                 R.scan('tstring_content', '')
     assert_equal ["heredoc\n"],
-                 R.scan('heredoc_content', "<<EOS\nheredoc\nEOS")
+                 R.scan('tstring_content', "<<EOS\nheredoc\nEOS")
     assert_equal ["heredoc\n"],
-                 R.scan('heredoc_content', "<<EOS\nheredoc\nEOS\n")
+                 R.scan('tstring_content', "<<EOS\nheredoc\nEOS\n")
     assert_equal ["heredoc \n"],
-                 R.scan('heredoc_content', "<<EOS\nheredoc \nEOS \n")
+                 R.scan('tstring_content', "<<EOS\nheredoc \nEOS \n")
     assert_equal ["heredoc\n"],
-                 R.scan('heredoc_content', "<<-EOS\nheredoc\n\tEOS \n")
+                 R.scan('tstring_content', "<<-EOS\nheredoc\n\tEOS \n")
   end
 
   def test_heredoc_end