kotovalexarian-likes-github
/
ruby--ruby
mirror of https://github.com/ruby/ruby.git
1
0
Fork 0
Code Releases Activity

* parse.y [ripper]: do not delay dispatching. 

* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6936 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
aamine 2004-09-20 07:59:30 +00:00
parent 273d0344a8
commit b2a700f13e
3 changed files with 43 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ChangeLog
View File

@ -1,3 +1,9 @@
Mon Sep 20 16:58:16 2004 Minero Aoki <aamine@loveruby.net>
* parse.y [ripper]: do not delay dispatching.
* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.
Mon Sep 20 15:17:47 2004 Minero Aoki <aamine@loveruby.net>
* ext/ripper/lib/ripper/tokenizer.rb: new file.

6
ext/ripper/lib/ripper/tokenizer.rb
View File

@ -24,11 +24,13 @@ class Ripper
def tokenize
@tokens = []
parse
@tokens
@tokens.sort_by {|tok, pos| pos }.map {|tok,| tok }
end
private
def on__scan(type, tok)
@tokens.push tok
@tokens.push [tok, [lineno(),column()]]
end
end

86
parse.y
View File

@ -145,9 +145,9 @@ struct parser_params {
int parser_ruby_sourceline;
VALUE parser_ruby_sourcefile;
char *tokp;
int current_t;
int in_heredoc;
VALUE delayed;
int delayed_line;
int delayed_col;
#endif
};
@ -4206,46 +4206,26 @@ ripper_dispatch_scan_event(parser, t)
if (lex_p == parser->tokp) return;
str = rb_str_new(parser->tokp, lex_p - parser->tokp);
event = ripper_token2eventid(t);
if (parser->in_heredoc) {
int col = parser->tokp - lex_pbeg;
rb_ary_push(parser->delayed, ID2SYM(event));
rb_ary_push(parser->delayed, str);
rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline));
rb_ary_push(parser->delayed, INT2NUM(col));
ripper_flush(parser);
return;
}
ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
ripper_dispatch1(parser, event, str);
yylval.val = ripper_dispatch1(parser, event, str);
ripper_flush(parser);
}
static void
ripper_dispatch_delayed_events(parser)
ripper_dispatch_delayed_token(parser, t)
struct parser_params *parser;
int t;
{
long i;
int saved_line;
char *saved_tokp;
int saved_line = ruby_sourceline;
char *saved_tokp = parser->tokp;
ID event = ripper_token2eventid(t);
VALUE str = parser->delayed;
if (RARRAY(parser->delayed)->len == 0) return;
if (RARRAY(parser->delayed)->len % 4 != 0)
rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0");
saved_line = ruby_sourceline;
saved_tokp = parser->tokp;
i = 0;
while (i < RARRAY(parser->delayed)->len) {
VALUE event, str, vline, vcol;
event = RARRAY(parser->delayed)->ptr[i++];
str = RARRAY(parser->delayed)->ptr[i++];
vline = RARRAY(parser->delayed)->ptr[i++];
vcol = RARRAY(parser->delayed)->ptr[i++];
ruby_sourceline = NUM2INT(vline);
parser->tokp = lex_pbeg + NUM2INT(vcol);
ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str));
ripper_dispatch1(parser, SYM2ID(event), str);
}
rb_ary_clear(parser->delayed);
ruby_sourceline = parser->delayed_line;
parser->tokp = lex_pbeg + parser->delayed_col;
ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str));
yylval.val = ripper_dispatch1(parser, event, str);
parser->delayed = Qnil;
ruby_sourceline = saved_line;
parser->tokp = saved_tokp;
}
@ -4486,9 +4466,17 @@ parser_nextc(parser)
}
#ifdef RIPPER
if (parser->tokp < lex_pend) {
if (!parser->current_t)
rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t");
ripper_dispatch_scan_event(parser, parser->current_t);
if (NIL_P(parser->delayed)) {
parser->delayed = rb_str_buf_new(1024);
rb_str_buf_cat(parser->delayed,
parser->tokp, lex_pend - parser->tokp);
parser->delayed_line = ruby_sourceline;
parser->delayed_col = parser->tokp - lex_pbeg;
}
else {
rb_str_buf_cat(parser->delayed,
parser->tokp, lex_pend - parser->tokp);
}
}
#endif
if (heredoc_end > 0) {
@ -4939,9 +4927,6 @@ parser_parse_string(parser, quote)
if (func == -1) return tSTRING_END;
c = nextc();
if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
#ifdef RIPPER
parser->current_t = ' ';
#endif
do {c = nextc();} while (ISSPACE(c));
space = 1;
}
@ -5057,6 +5042,8 @@ parser_heredoc_restore(parser, here)
VALUE line;
#ifdef RIPPER
if (!NIL_P(parser->delayed))
ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
lex_goto_eol(parser);
ripper_dispatch_scan_event(parser, tHEREDOC_END);
#endif
@ -5071,7 +5058,6 @@ parser_heredoc_restore(parser, here)
rb_gc_force_recycle((VALUE)here);
#ifdef RIPPER
ripper_flush(parser);
parser->in_heredoc = Qfalse;
#endif
}
@ -5222,13 +5208,7 @@ parser_yylex(parser)
if (lex_strterm) {
int token;
#ifdef RIPPER
parser->current_t = tSTRING_CONTENT;
#endif
if (nd_type(lex_strterm) == NODE_HEREDOC) {
#ifdef RIPPER
parser->in_heredoc = Qtrue;
#endif
token = here_document(lex_strterm);
if (token == tSTRING_END) {
lex_strterm = 0;
@ -6444,11 +6424,12 @@ yylex(p)
#endif
t = parser_yylex(parser);
#ifdef RIPPER
parser->current_t = 0;
if (!NIL_P(parser->delayed)) {
ripper_dispatch_delayed_token(parser, t);
return t;
}
if (t != 0)
ripper_dispatch_scan_event(parser, t);
if ((t == '\n' && !parser->in_heredoc) || (t == 0))
ripper_dispatch_delayed_events(parser);
#endif
return t;
@ -8137,9 +8118,8 @@ parser_initialize(parser)
parser->parser_lex_p = 0;
parser->parser_lex_pend = 0;
#ifdef RIPPER
parser->current_t = 0;
parser->in_heredoc = Qfalse;
parser->delayed = rb_ary_new();
parser->parser_ruby_sourcefile = Qnil;
parser->delayed = Qnil;
#endif
}