* parse.y [ripper]: do not delay dispatching.

* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6936 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,9 @@
+Mon Sep 20 16:58:16 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* parse.y [ripper]: do not delay dispatching.
+
+	* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.
+
 Mon Sep 20 15:17:47 2004  Minero Aoki  <aamine@loveruby.net>
 
 	* ext/ripper/lib/ripper/tokenizer.rb: new file.

ext/ripper/lib/ripper/tokenizer.rb

@@ -24,11 +24,13 @@ class Ripper
     def tokenize
       @tokens = []
       parse
-      @tokens
+      @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok }
     end
 
+    private
+
     def on__scan(type, tok)
-      @tokens.push tok
+      @tokens.push [tok, [lineno(),column()]]
     end
   end
 

parse.y

@@ -145,9 +145,9 @@ struct parser_params {
     int parser_ruby_sourceline;
     VALUE parser_ruby_sourcefile;
     char *tokp;
-    int current_t;
-    int in_heredoc;
     VALUE delayed;
+    int delayed_line;
+    int delayed_col;
 #endif
 };
 
@@ -4206,46 +4206,26 @@ ripper_dispatch_scan_event(parser, t)
     if (lex_p == parser->tokp) return;
     str = rb_str_new(parser->tokp, lex_p - parser->tokp);
     event = ripper_token2eventid(t);
-    if (parser->in_heredoc) {
-	int col = parser->tokp - lex_pbeg;
-	rb_ary_push(parser->delayed, ID2SYM(event));
-	rb_ary_push(parser->delayed, str);
-	rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline));
-	rb_ary_push(parser->delayed, INT2NUM(col));
-	ripper_flush(parser);
-	return;
-    }
     ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
-    ripper_dispatch1(parser, event, str);
+    yylval.val = ripper_dispatch1(parser, event, str);
     ripper_flush(parser);
 }
 
 static void
-ripper_dispatch_delayed_events(parser)
+ripper_dispatch_delayed_token(parser, t)
     struct parser_params *parser;
+    int t;
 {
-    long i;
+    int saved_line = ruby_sourceline;
-    int saved_line;
+    char *saved_tokp = parser->tokp;
-    char *saved_tokp;
+    ID event = ripper_token2eventid(t);
+    VALUE str = parser->delayed;
 
-    if (RARRAY(parser->delayed)->len == 0) return;
+    ruby_sourceline = parser->delayed_line;
-    if (RARRAY(parser->delayed)->len % 4 != 0)
+    parser->tokp = lex_pbeg + parser->delayed_col;
-    	rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0");
+    ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
-    saved_line = ruby_sourceline;
+    yylval.val = ripper_dispatch1(parser, event, str);
-    saved_tokp = parser->tokp;
+    parser->delayed = Qnil;
-    i = 0;
-    while (i < RARRAY(parser->delayed)->len) {
-	VALUE event, str, vline, vcol;
-	event = RARRAY(parser->delayed)->ptr[i++];
-	str = RARRAY(parser->delayed)->ptr[i++];
-	vline = RARRAY(parser->delayed)->ptr[i++];
-	vcol = RARRAY(parser->delayed)->ptr[i++];
-	ruby_sourceline = NUM2INT(vline);
-	parser->tokp = lex_pbeg + NUM2INT(vcol);
-	ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str));
-	ripper_dispatch1(parser, SYM2ID(event), str);
-    }
-    rb_ary_clear(parser->delayed);
     ruby_sourceline = saved_line;
     parser->tokp = saved_tokp;
 }
@@ -4486,9 +4466,17 @@ parser_nextc(parser)
             }
 #ifdef RIPPER
 	    if (parser->tokp < lex_pend) {
-		if (!parser->current_t)
+		if (NIL_P(parser->delayed)) {
-		    rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t");
+		    parser->delayed = rb_str_buf_new(1024);
-		ripper_dispatch_scan_event(parser, parser->current_t);
+		    rb_str_buf_cat(parser->delayed,
+				   parser->tokp, lex_pend - parser->tokp);
+		    parser->delayed_line = ruby_sourceline;
+		    parser->delayed_col = parser->tokp - lex_pbeg;
+		}
+		else {
+		    rb_str_buf_cat(parser->delayed,
+				   parser->tokp, lex_pend - parser->tokp);
+		}
 	    }
 #endif
 	    if (heredoc_end > 0) {
@@ -4939,9 +4927,6 @@ parser_parse_string(parser, quote)
     if (func == -1) return tSTRING_END;
     c = nextc();
     if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
-#ifdef RIPPER
-	parser->current_t = ' ';
-#endif
 	do {c = nextc();} while (ISSPACE(c));
 	space = 1;
     }
@@ -5057,6 +5042,8 @@ parser_heredoc_restore(parser, here)
     VALUE line;
 
 #ifdef RIPPER
+    if (!NIL_P(parser->delayed))
+	ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
     lex_goto_eol(parser);
     ripper_dispatch_scan_event(parser, tHEREDOC_END);
 #endif
@@ -5071,7 +5058,6 @@ parser_heredoc_restore(parser, here)
     rb_gc_force_recycle((VALUE)here);
 #ifdef RIPPER
     ripper_flush(parser);
-    parser->in_heredoc = Qfalse;
 #endif
 }
 
@@ -5222,13 +5208,7 @@ parser_yylex(parser)
 
     if (lex_strterm) {
 	int token;
-#ifdef RIPPER
-	parser->current_t = tSTRING_CONTENT;
-#endif
 	if (nd_type(lex_strterm) == NODE_HEREDOC) {
-#ifdef RIPPER
-	    parser->in_heredoc = Qtrue;
-#endif
 	    token = here_document(lex_strterm);
 	    if (token == tSTRING_END) {
 		lex_strterm = 0;
@@ -6444,11 +6424,12 @@ yylex(p)
 #endif
     t = parser_yylex(parser);
 #ifdef RIPPER
-    parser->current_t = 0;
+    if (!NIL_P(parser->delayed)) {
+	ripper_dispatch_delayed_token(parser, t);
+	return t;
+    }
     if (t != 0)
 	ripper_dispatch_scan_event(parser, t);
-    if ((t == '\n' && !parser->in_heredoc) || (t == 0))
-        ripper_dispatch_delayed_events(parser);
 #endif
 
     return t;
@@ -8137,9 +8118,8 @@ parser_initialize(parser)
     parser->parser_lex_p = 0;
     parser->parser_lex_pend = 0;
 #ifdef RIPPER
-    parser->current_t = 0;
+    parser->parser_ruby_sourcefile = Qnil;
-    parser->in_heredoc = Qfalse;
+    parser->delayed = Qnil;
-    parser->delayed = rb_ary_new();
 #endif
 }