kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

* transcode_data.h (rb_transcoding): add fields: writebuf_off,

Browse source 
writebuf_len and writebuf.
  (TRANSCODING_WRITEBUF): new macro.

* transcode.c (transcode_restartable0): output until output buffer is
  really full.
  (rb_transcoding_open_by_transcoder): initialize writebuf_len,
  writebuf_off and writebuf.
  (rb_transcoding_close): finalize writebuf.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@18573 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
akr 2008-08-13 08:28:39 +00:00
parent 8f1e7a37e5
commit 04f7390ca8
3 changed files with 106 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

12
ChangeLog
View file

@ -1,3 +1,15 @@
Wed Aug 13 17:26:44 2008 Tanaka Akira <akr@fsij.org>
* transcode_data.h (rb_transcoding): add fields: writebuf_off,
writebuf_len and writebuf.
(TRANSCODING_WRITEBUF): new macro.
* transcode.c (transcode_restartable0): output until output buffer is
really full.
(rb_transcoding_open_by_transcoder): initialize writebuf_len,
writebuf_off and writebuf.
(rb_transcoding_close): finalize writebuf.
Wed Aug 13 17:18:37 2008 Koichi Sasada <ko1@atdot.net> Wed Aug 13 17:18:37 2008 Koichi Sasada <ko1@atdot.net>
* thread.c (rb_thread_check_ints): added. please note that * thread.c (rb_thread_check_ints): added. please note that

107
transcode.c
View file

@ -405,6 +405,13 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
return ret; \ return ret; \
resume_label ## num:; \ resume_label ## num:; \
} while (0) } while (0)
#define SUSPEND_OBUF(num) \
do { \
while (out_stop - out_p < 1) { SUSPEND(transcode_obuf_full, num); } \
} while (0)
#define writebuf_len (tc->writebuf_len)
#define writebuf_off (tc->writebuf_off)
switch (tc->resume_position) { switch (tc->resume_position) {
case 0: break; case 0: break;
@ -422,6 +429,15 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
case 12: goto resume_label12; case 12: goto resume_label12;
case 13: goto resume_label13; case 13: goto resume_label13;
case 14: goto resume_label14; case 14: goto resume_label14;
case 15: goto resume_label15;
case 16: goto resume_label16;
case 17: goto resume_label17;
case 18: goto resume_label18;
case 19: goto resume_label19;
case 20: goto resume_label20;
case 21: goto resume_label21;
case 22: goto resume_label22;
case 23: goto resume_label23;
} }
while (1) { while (1) {
@ -446,8 +462,7 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
follow_info: follow_info:
switch (next_info & 0x1F) { switch (next_info & 0x1F) {
case NOMAP: /* xxx: copy last byte only? */ case NOMAP: /* xxx: copy last byte only? */
while (out_stop - out_p < 1) { SUSPEND(transcode_obuf_full, 3); } SUSPEND_OBUF(3); *out_p++ = next_byte;
*out_p++ = next_byte;
continue; continue;
case 0x00: case 0x04: case 0x08: case 0x0C: case 0x00: case 0x04: case 0x08: case 0x0C:
case 0x10: case 0x14: case 0x18: case 0x1C: case 0x10: case 0x14: case 0x18: case 0x1C:
@ -462,26 +477,22 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
case ZERObt: /* drop input */ case ZERObt: /* drop input */
continue; continue;
case ONEbt: case ONEbt:
while (out_stop - out_p < 1) { SUSPEND(transcode_obuf_full, 9); } SUSPEND_OBUF(9); *out_p++ = getBT1(next_info);
*out_p++ = getBT1(next_info);
continue; continue;
case TWObt: case TWObt:
while (out_stop - out_p < 2) { SUSPEND(transcode_obuf_full, 10); } SUSPEND_OBUF(10); *out_p++ = getBT1(next_info);
*out_p++ = getBT1(next_info); SUSPEND_OBUF(21); *out_p++ = getBT2(next_info);
*out_p++ = getBT2(next_info);
continue; continue;
case THREEbt: case THREEbt:
while (out_stop - out_p < 3) { SUSPEND(transcode_obuf_full, 11); } SUSPEND_OBUF(11); *out_p++ = getBT1(next_info);
*out_p++ = getBT1(next_info); SUSPEND_OBUF(15); *out_p++ = getBT2(next_info);
*out_p++ = getBT2(next_info); SUSPEND_OBUF(16); *out_p++ = getBT3(next_info);
*out_p++ = getBT3(next_info);
continue; continue;
case FOURbt: case FOURbt:
while (out_stop - out_p < 4) { SUSPEND(transcode_obuf_full, 12); } SUSPEND_OBUF(12); *out_p++ = getBT0(next_info);
*out_p++ = getBT0(next_info); SUSPEND_OBUF(17); *out_p++ = getBT1(next_info);
*out_p++ = getBT1(next_info); SUSPEND_OBUF(18); *out_p++ = getBT2(next_info);
*out_p++ = getBT2(next_info); SUSPEND_OBUF(19); *out_p++ = getBT3(next_info);
*out_p++ = getBT3(next_info);
continue; continue;
case FUNii: case FUNii:
next_info = (VALUE)(*tr->func_ii)(tc, next_info); next_info = (VALUE)(*tr->func_ii)(tc, next_info);
@ -495,16 +506,36 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
goto follow_info; goto follow_info;
} }
case FUNio: case FUNio:
while (out_stop - out_p < tr->max_output) { SUSPEND(transcode_obuf_full, 13); } SUSPEND_OBUF(13);
if (tr->max_output <= out_stop - out_p)
out_p += (VALUE)(*tr->func_io)(tc, next_info, out_p); out_p += (VALUE)(*tr->func_io)(tc, next_info, out_p);
else {
writebuf_len = (VALUE)(*tr->func_io)(tc, next_info, TRANSCODING_WRITEBUF(tc));
writebuf_off = 0;
while (writebuf_off < writebuf_len) {
SUSPEND_OBUF(20);
*out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
}
}
break; break;
case FUNso: case FUNso:
{ {
const unsigned char *char_start; const unsigned char *char_start;
size_t char_len; size_t char_len;
while (out_stop - out_p < tr->max_output) { SUSPEND(transcode_obuf_full, 14); } SUSPEND_OBUF(14);
if (tr->max_output <= out_stop - out_p) {
char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
out_p += (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, out_p); out_p += (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, out_p);
}
else {
char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
writebuf_len = (VALUE)(*tr->func_so)(tc, char_start, (size_t)char_len, TRANSCODING_WRITEBUF(tc));
writebuf_off = 0;
while (writebuf_off < writebuf_len) {
SUSPEND_OBUF(22);
*out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
}
}
break; break;
} }
case INVALID: case INVALID:
@ -544,14 +575,24 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
/* cleanup */ /* cleanup */
if (tr->finish_func) { if (tr->finish_func) {
while (out_stop - out_p < tr->max_output) { SUSPEND_OBUF(4);
SUSPEND(transcode_obuf_full, 4); if (tr->max_output <= out_stop - out_p) {
}
out_p += tr->finish_func(tc, out_p); out_p += tr->finish_func(tc, out_p);
} }
else {
writebuf_len = tr->finish_func(tc, TRANSCODING_WRITEBUF(tc));
writebuf_off = 0;
while (writebuf_off < writebuf_len) {
SUSPEND_OBUF(23);
*out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
}
}
}
while (1) while (1)
SUSPEND(transcode_finished, 6); SUSPEND(transcode_finished, 6);
#undef SUSPEND #undef SUSPEND
#undef writebuf_len
#undef writebuf_off
} }
static rb_trans_result_t static rb_trans_result_t
@ -592,9 +633,14 @@ rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags)
tc->resume_position = 0; tc->resume_position = 0;
tc->recognized_len = 0; tc->recognized_len = 0;
tc->readagain_len = 0; tc->readagain_len = 0;
tc->writebuf_len = 0;
tc->writebuf_off = 0;
if (sizeof(tc->readbuf.ary) < tr->max_input) { if (sizeof(tc->readbuf.ary) < tr->max_input) {
tc->readbuf.ptr = xmalloc(tr->max_input); tc->readbuf.ptr = xmalloc(tr->max_input);
} }
if (sizeof(tc->writebuf.ary) < tr->max_output) {
tc->writebuf.ptr = xmalloc(tr->max_output);
}
return tc; return tc;
} }
@ -629,6 +675,8 @@ rb_transcoding_close(rb_transcoding *tc)
const rb_transcoder *tr = tc->transcoder; const rb_transcoder *tr = tc->transcoder;
if (sizeof(tc->readbuf.ary) < tr->max_input) if (sizeof(tc->readbuf.ary) < tr->max_input)
xfree(tc->readbuf.ptr); xfree(tc->readbuf.ptr);
if (sizeof(tc->writebuf.ary) < tr->max_output)
xfree(tc->writebuf.ptr);
xfree(tc); xfree(tc);
} }
@ -1384,9 +1432,6 @@ check_econv(VALUE self)
* input_buffer and output_buffer should be a string. * input_buffer and output_buffer should be a string.
* output_bufsize and flags should be an integer. * output_bufsize and flags should be an integer.
* *
* output_bufsize should be greater than or equal to the value of
* Encoding::Converter#max_output.
*
* primitive_convert convert the content of input_buffer from beginning * primitive_convert convert the content of input_buffer from beginning
* and store the result into output_buffer. * and store the result into output_buffer.
* *
@ -1410,10 +1455,14 @@ check_econv(VALUE self)
* p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"] * p [ret, src, dst] #=> [:finished, "", "\x00p\x00i"]
* *
* ec = Encoding::Converter.new("UTF-8", "UTF-16BE") * ec = Encoding::Converter.new("UTF-8", "UTF-16BE")
* ret = ec.primitive_convert(src="pi", dst="", 4) * ret = ec.primitive_convert(src="pi", dst="", 1)
* p [ret, src, dst] # [:obuf_full, "", "\x00p"] * p [ret, src, dst] #=> [:obuf_full, "i", "\x00"]
* ret = ec.primitive_convert(src, dst="", 4) * ret = ec.primitive_convert(src, dst="", 1)
* p [ret, src, dst] # [:finished, "", "\x00i"] * p [ret, src, dst] #=> [:obuf_full, "", "p"]
* ret = ec.primitive_convert(src, dst="", 1)
* p [ret, src, dst] #=> [:obuf_full, "", "\x00"]
* ret = ec.primitive_convert(src, dst="", 1)
* p [ret, src, dst] #=> [:finished, "", "i"]
* *
*/ */
static VALUE static VALUE
@ -1466,8 +1515,6 @@ econv_primitive_convert(int argc, VALUE *argv, VALUE self)
* max_output -> int * max_output -> int
* *
* returns the maximum length of output unit in bytes. * returns the maximum length of output unit in bytes.
*
* This value is the minimum value of output_bufsize argument of primitive_convert.
*/ */
static VALUE static VALUE
econv_max_output(VALUE self) econv_max_output(VALUE self)

14
transcode_data.h
View file

@ -69,19 +69,31 @@ typedef struct rb_transcoding {
const BYTE_LOOKUP *next_table; const BYTE_LOOKUP *next_table;
VALUE next_info; VALUE next_info;
unsigned char next_byte; unsigned char next_byte;
int recognized_len; /* already interpreted */ int recognized_len; /* already interpreted */
int readagain_len; /* not yet interpreted */ int readagain_len; /* not yet interpreted */
union { union {
unsigned char ary[8]; /* max_input <= sizeof(ary) */ unsigned char ary[8]; /* max_input <= sizeof(ary) */
unsigned char *ptr; /* length is max_input */ unsigned char *ptr; /* length: max_input */
} readbuf; /* recognized_len + readagain_len used */ } readbuf; /* recognized_len + readagain_len used */
int writebuf_off;
int writebuf_len;
union {
unsigned char ary[8]; /* max_output <= sizeof(ary) */
unsigned char *ptr; /* length: max_output */
} writebuf;
unsigned char stateful[256]; /* opaque data for stateful encoding */ unsigned char stateful[256]; /* opaque data for stateful encoding */
} rb_transcoding; } rb_transcoding;
#define TRANSCODING_READBUF(tc) \ #define TRANSCODING_READBUF(tc) \
((tc)->transcoder->max_input <= sizeof((tc)->readbuf.ary) ? \ ((tc)->transcoder->max_input <= sizeof((tc)->readbuf.ary) ? \
(tc)->readbuf.ary : \ (tc)->readbuf.ary : \
(tc)->readbuf.ptr) (tc)->readbuf.ptr)
#define TRANSCODING_WRITEBUF(tc) \
((tc)->transcoder->max_output <= sizeof((tc)->writebuf.ary) ? \
(tc)->writebuf.ary : \
(tc)->writebuf.ptr)
/* static structure, one per supported encoding pair */ /* static structure, one per supported encoding pair */
struct rb_transcoder { struct rb_transcoder {