kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

* transcode.c: new file to provide encoding conversion features.

Browse source 
code contributed by Martin Duerst.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@14172 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
matz 2007-12-10 05:01:47 +00:00
parent 38a24d73c8
commit 7ded13f54b
8 changed files with 3797 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ChangeLog
View file

@ -1,3 +1,8 @@
Mon Dec 10 14:00:43 2007 Yukihiro Matsumoto <matz@ruby-lang.org>
* transcode.c: new file to provide encoding conversion features.
code contributed by Martin Duerst.
Mon Dec 10 13:50:33 2007 Nobuyoshi Nakada <nobu@ruby-lang.org>
* re.c (rb_reg_search): return byte offset. [ruby-dev:32452]

7
common.mk
View file

@ -65,6 +65,8 @@ COMMONOBJS = array.$(OBJEXT) \
string.$(OBJEXT) \
struct.$(OBJEXT) \
time.$(OBJEXT) \
transcode.$(OBJEXT) \
transcode_data_iso_8859.$(OBJEXT) \
util.$(OBJEXT) \
variable.$(OBJEXT) \
version.$(OBJEXT) \
@ -530,7 +532,7 @@ sprintf.$(OBJEXT): {$(VPATH)}sprintf.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
st.$(OBJEXT): {$(VPATH)}st.c {$(VPATH)}config.h {$(VPATH)}st.h {$(VPATH)}defines.h
string.$(OBJEXT): {$(VPATH)}string.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
{$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \
{$(VPATH)}re.h {$(VPATH)}regex.h {$(VPATH)}encoding.h
{$(VPATH)}re.h {$(VPATH)}regex.h {$(VPATH)}encoding.h
struct.$(OBJEXT): {$(VPATH)}struct.c {$(VPATH)}ruby.h {$(VPATH)}config.h \
{$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h
thread.$(OBJEXT): {$(VPATH)}thread.c {$(VPATH)}eval_intern.h \
@ -540,6 +542,9 @@ thread.$(OBJEXT): {$(VPATH)}thread.c {$(VPATH)}eval_intern.h \
{$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \
{$(VPATH)}node.h {$(VPATH)}util.h \
{$(VPATH)}signal.h {$(VPATH)}st.h {$(VPATH)}dln.h
transcode.$(OBJEXT): {$(VPATH)}transcode.c {$(VPATH)}transcode_data.h {$(VPATH)}ruby.h {$(VPATH)}config.h \
{$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h {$(VPATH)}encoding.h
transcode_data_iso_8859.$(OBJEXT): {$(VPATH)}transcode_data_iso_8859.c {$(VPATH)}transcode_data.h
cont.$(OBJEXT): {$(VPATH)}cont.c {$(VPATH)}eval_intern.h \
{$(VPATH)}ruby.h {$(VPATH)}vm_core.h {$(VPATH)}id.h {$(VPATH)}config.h \
{$(VPATH)}defines.h {$(VPATH)}intern.h {$(VPATH)}missing.h \

2
inits.c
View file

@ -16,6 +16,7 @@ void Init_Array(void);
void Init_Bignum(void);
void Init_Binding(void);
void Init_Comparable(void);
void Init_transcode(void);
void Init_Dir(void);
void Init_Enumerable(void);
void Init_Enumerator(void);
@ -77,6 +78,7 @@ rb_call_inits()
Init_Struct();
Init_Regexp();
Init_pack();
Init_transcode();
Init_marshal();
Init_Range();
Init_IO();

4
string.c
View file

@ -179,7 +179,7 @@ str_alloc(VALUE klass)
return (VALUE)str;
}
static VALUE
VALUE
str_new(VALUE klass, const char *ptr, long len)
{
VALUE str;
@ -625,7 +625,7 @@ str_modifiable(VALUE str)
rb_raise(rb_eSecurityError, "Insecure: can't modify string");
}
static int
int
str_independent(VALUE str)
{
str_modifiable(str);

44
test/ruby/test_transcode.rb Normal file
View file

@ -0,0 +1,44 @@
# -*- encoding: US-ASCII -*- # make sure this runs in binary mode
class String
# different name, because we should be able to remove this later
def fix_encoding (encoding)
force_encoding(encoding)
end
end
require 'test/unit'
class TestConvert < Test::Unit::TestCase
def test_can_call
# we don't have semantics for conversion without attribute yet
# maybe 'convert to UTF-8' would be nice :-)
assert_raise(ArgumentError) { 'abc'.encode }
assert_raise(ArgumentError) { 'abc'.encode! }
assert_raise(ArgumentError) { 'abc'.force_encoding('Shift_JIS').encode('UTF-8') } # temporary
assert_raise(ArgumentError) { 'abc'.force_encoding('Shift_JIS').encode!('UTF-8') } # temporary
assert_raise(ArgumentError) { 'abc'.encode('foo', 'bar') }
assert_raise(ArgumentError) { 'abc'.encode!('foo', 'bar') }
assert_raise(ArgumentError) { 'abc'.force_encoding('utf-8').encode('foo') }
assert_raise(ArgumentError) { 'abc'.force_encoding('utf-8').encode!('foo') }
assert_equal('abc'.force_encoding('utf-8').encode('iso-8859-1'), 'abc') # temporary, fix encoding
assert_equal("D\xFCrst".force_encoding('iso-8859-1').encode('utf-8').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-1').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-2').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-3').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-4').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-9').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-10').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-13').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-14').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\xFCrst".encode('utf-8', 'iso-8859-15').fix_encoding('utf-8'), "D\u00FCrst")
assert_equal("D\u00FCrst".encode('iso-8859-1'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-2'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-3'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-4'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-9'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-10'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-13'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-14'), "D\xFCrst")
assert_equal("D\u00FCrst".encode('iso-8859-15'), "D\xFCrst")
end
end

438
transcode.c Normal file
View file

@ -0,0 +1,438 @@
/**********************************************************************
transcode.c -
$Author: duerst $
$Date: 2007-10-30 16:10:22 +0900 (Tue, 30 Oct 2007) $
created at: Tue Oct 30 16:10:22 JST 2007
Copyright (C) 2007 Martin Duerst
**********************************************************************/
#include "ruby/ruby.h"
#include "ruby/encoding.h"
#include "transcode_data.h"
/*
* prototypes and macros copied from string.c (temporarily !!!)
*/
VALUE str_new(VALUE klass, const char *ptr, long len);
int str_independent(VALUE str);
#define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,ELTS_SHARED|STR_ASSOC))
#define STR_NOEMBED FL_USER1
#define STR_ASSOC FL_USER3
#define STR_SET_NOEMBED(str) do {\
FL_SET(str, STR_NOEMBED);\
STR_SET_EMBED_LEN(str, 0);\
} while (0)
#define STR_UNSET_NOCAPA(s) do {\
if (FL_TEST(s,STR_NOEMBED)) FL_UNSET(s,(ELTS_SHARED|STR_ASSOC));\
} while (0)
#define STR_SET_EMBED_LEN(str, n) do { \
long tmp_n = (n);\
RBASIC(str)->flags &= ~RSTRING_EMBED_LEN_MASK;\
RBASIC(str)->flags |= (tmp_n) << RSTRING_EMBED_LEN_SHIFT;\
} while (0)
#define STR_SET_LEN(str, n) do { \
if (STR_EMBED_P(str)) {\
STR_SET_EMBED_LEN(str, n);\
}\
else {\
RSTRING(str)->as.heap.len = (n);\
}\
} while (0)
#define STR_EMBED_P(str) (!FL_TEST(str, STR_NOEMBED))
#define RESIZE_CAPA(str,capacity) do {\
if (STR_EMBED_P(str)) {\
if ((capacity) > RSTRING_EMBED_LEN_MAX) {\
char *tmp = ALLOC_N(char, capacity+1);\
memcpy(tmp, RSTRING_PTR(str), RSTRING_LEN(str));\
RSTRING(str)->as.heap.ptr = tmp;\
RSTRING(str)->as.heap.len = RSTRING_LEN(str);\
STR_SET_NOEMBED(str);\
RSTRING(str)->as.heap.aux.capa = (capacity);\
}\
}\
else {\
REALLOC_N(RSTRING(str)->as.heap.ptr, char, (capacity)+1);\
if (!STR_NOCAPA_P(str))\
RSTRING(str)->as.heap.aux.capa = (capacity);\
}\
} while (0)
/* end of copied prototypes and macros */
/*
* Dispatch data and logic
*/
/* extern declarations, should use some include file here */
extern const BYTE_LOOKUP from_ISO_8859_1;
extern const BYTE_LOOKUP from_ISO_8859_2;
extern const BYTE_LOOKUP from_ISO_8859_3;
extern const BYTE_LOOKUP from_ISO_8859_4;
extern const BYTE_LOOKUP from_ISO_8859_5;
extern const BYTE_LOOKUP from_ISO_8859_6;
extern const BYTE_LOOKUP from_ISO_8859_7;
extern const BYTE_LOOKUP from_ISO_8859_8;
extern const BYTE_LOOKUP from_ISO_8859_9;
extern const BYTE_LOOKUP from_ISO_8859_10;
extern const BYTE_LOOKUP from_ISO_8859_11;
extern const BYTE_LOOKUP from_ISO_8859_13;
extern const BYTE_LOOKUP from_ISO_8859_14;
extern const BYTE_LOOKUP from_ISO_8859_15;
extern const BYTE_LOOKUP to_ISO_8859_1;
extern const BYTE_LOOKUP to_ISO_8859_2;
extern const BYTE_LOOKUP to_ISO_8859_3;
extern const BYTE_LOOKUP to_ISO_8859_4;
extern const BYTE_LOOKUP to_ISO_8859_5;
extern const BYTE_LOOKUP to_ISO_8859_6;
extern const BYTE_LOOKUP to_ISO_8859_7;
extern const BYTE_LOOKUP to_ISO_8859_8;
extern const BYTE_LOOKUP to_ISO_8859_9;
extern const BYTE_LOOKUP to_ISO_8859_10;
extern const BYTE_LOOKUP to_ISO_8859_11;
extern const BYTE_LOOKUP to_ISO_8859_13;
extern const BYTE_LOOKUP to_ISO_8859_14;
extern const BYTE_LOOKUP to_ISO_8859_15;
/* declarations probably need to go into separate header file, e.g. transcode.h */
/* static structure, one per supported encoding pair */
typedef struct {
const char *from_encoding;
const char *to_encoding;
BYTE_LOOKUP *conv_tree_start;
int max_output;
int from_utf8;
} transcoder;
/* todo: dynamic structure, one per conversion (stream) */
/* in the future, add some mechanism for dynamically adding stuff here */
#define MAX_TRANSCODERS 29 /* todo: fix: this number has to be adjusted by hand */
static transcoder transcoder_table[MAX_TRANSCODERS];
/* not sure why it's not possible to do relocatable initializations */
/* maybe the code here can be removed (changed to simple initialization) */
/* if we move this to another file???? */
static void
register_transcoder (const char *from_e, const char *to_e,
const BYTE_LOOKUP *tree_start, int max_output, int from_utf8)
{
static int n = 0;
if (n >= MAX_TRANSCODERS) {
/* we are initializing, is it okay to use rb_raise here? */
rb_raise(rb_eRuntimeError /*change exception*/, "not enough transcoder slots");
}
transcoder_table[n].from_encoding = from_e;
transcoder_table[n].to_encoding = to_e;
transcoder_table[n].conv_tree_start = (BYTE_LOOKUP *)tree_start;
transcoder_table[n].max_output = max_output;
transcoder_table[n].from_utf8 = from_utf8;
n++;
}
static void
init_transcoder_table (void)
{
register_transcoder("ISO-8859-1", "UTF-8", &from_ISO_8859_1, 2, 0);
register_transcoder("ISO-8859-2", "UTF-8", &from_ISO_8859_2, 2, 0);
register_transcoder("ISO-8859-3", "UTF-8", &from_ISO_8859_3, 2, 0);
register_transcoder("ISO-8859-4", "UTF-8", &from_ISO_8859_4, 2, 0);
register_transcoder("ISO-8859-5", "UTF-8", &from_ISO_8859_5, 3, 0);
register_transcoder("ISO-8859-6", "UTF-8", &from_ISO_8859_6, 2, 0);
register_transcoder("ISO-8859-7", "UTF-8", &from_ISO_8859_7, 3, 0);
register_transcoder("ISO-8859-8", "UTF-8", &from_ISO_8859_8, 3, 0);
register_transcoder("ISO-8859-9", "UTF-8", &from_ISO_8859_9, 2, 0);
register_transcoder("ISO-8859-10", "UTF-8", &from_ISO_8859_10, 3, 0);
register_transcoder("ISO-8859-11", "UTF-8", &from_ISO_8859_11, 3, 0);
register_transcoder("ISO-8859-13", "UTF-8", &from_ISO_8859_13, 3, 0);
register_transcoder("ISO-8859-14", "UTF-8", &from_ISO_8859_14, 3, 0);
register_transcoder("ISO-8859-15", "UTF-8", &from_ISO_8859_15, 3, 0);
register_transcoder("UTF-8", "ISO-8859-1", &to_ISO_8859_1, 1, 1);
register_transcoder("UTF-8", "ISO-8859-2", &to_ISO_8859_2, 1, 1);
register_transcoder("UTF-8", "ISO-8859-3", &to_ISO_8859_3, 1, 1);
register_transcoder("UTF-8", "ISO-8859-4", &to_ISO_8859_4, 1, 1);
register_transcoder("UTF-8", "ISO-8859-5", &to_ISO_8859_5, 1, 1);
register_transcoder("UTF-8", "ISO-8859-6", &to_ISO_8859_6, 1, 1);
register_transcoder("UTF-8", "ISO-8859-7", &to_ISO_8859_7, 1, 1);
register_transcoder("UTF-8", "ISO-8859-8", &to_ISO_8859_8, 1, 1);
register_transcoder("UTF-8", "ISO-8859-9", &to_ISO_8859_9, 1, 1);
register_transcoder("UTF-8", "ISO-8859-10", &to_ISO_8859_10, 1, 1);
register_transcoder("UTF-8", "ISO-8859-11", &to_ISO_8859_11, 1, 1);
register_transcoder("UTF-8", "ISO-8859-13", &to_ISO_8859_13, 1, 1);
register_transcoder("UTF-8", "ISO-8859-14", &to_ISO_8859_14, 1, 1);
register_transcoder("UTF-8", "ISO-8859-15", &to_ISO_8859_15, 1, 1);
register_transcoder(NULL, NULL, NULL, 0, 0);
}
static transcoder*
transcode_dispatch (char* from_encoding, char* to_encoding)
{
transcoder *candidate = transcoder_table;
for (candidate = transcoder_table; candidate->from_encoding; candidate++)
if (0==strcasecmp(from_encoding, candidate->from_encoding)
&& 0==strcasecmp(to_encoding, candidate->to_encoding))
break;
/* in the future, add multistep transcoding logic here */
return candidate->from_encoding ? candidate : NULL;
}
/* dynamic structure, one per conversion (similar to iconv_t) */
/* may carry conversion state (e.g. for iso-2022-jp) */
typedef struct {
VALUE ruby_string_dest; /* the String used as the conversion destination,
or NULL if something else is being converted */
} transcoding;
/*
* Transcoding engine logic
*/
static void
transcode_loop (unsigned char **in_pos, unsigned char **out_pos,
unsigned char *in_stop, unsigned char *out_stop,
transcoder *my_transcoder,
transcoding *my_transcoding)
{
unsigned char *input = *in_pos, *output = *out_pos;
unsigned char *in_p = *in_pos, *out_p = *out_pos;
BYTE_LOOKUP *conv_tree_start = my_transcoder->conv_tree_start;
BYTE_LOOKUP *next_table;
unsigned int next_offset;
unsigned int next_info;
unsigned char next_byte;
int from_utf8 = my_transcoder->from_utf8;
unsigned char *out_s = out_stop - my_transcoder->max_output + 1;
while (in_p < in_stop) {
unsigned char *char_start = in_p;
next_table = conv_tree_start;
if (out_p >= out_s) {
VALUE dest_string = my_transcoding->ruby_string_dest;
if (!dest_string) {
rb_raise(rb_eArgError /*@@@change exception*/, "Unable to obtain more space for transcoding");
}
else {
int len = (out_p - *out_pos);
int new_len = (len + my_transcoder->max_output) * 2;
RESIZE_CAPA(dest_string, new_len);
STR_SET_LEN(dest_string, new_len);
*out_pos = RSTRING_PTR(dest_string);
out_p = *out_pos + len;
out_s = *out_pos + new_len - my_transcoder->max_output;
}
}
next_byte = *in_p++;
follow_byte:
next_offset = next_table->base[next_byte];
next_info = (unsigned int)next_table->info[next_offset];
switch (next_info & 0x1F) {
case NOMAP:
*out_p++ = next_byte;
continue;
case 0x00: case 0x04: case 0x08: case 0x0C:
case 0x10: case 0x14: case 0x18: case 0x1C:
if (in_p >= in_stop) {
/* todo: deal with the case of backtracking */
/* todo: deal with incomplete input (streaming) */
goto illegal;
}
next_byte = *in_p++;
if (from_utf8) {
if ((next_byte&0xC0) == 0x80)
next_byte -= 0x80;
else
goto illegal;
}
next_table = (BYTE_LOOKUP*)next_info;
goto follow_byte;
/* maybe rewrite the following cases to use fallthrough???? */
case ZERObt: /* drop input */
continue;
case ONEbt:
*out_p++ = getBT1(next_info);
continue;
case TWObt:
*out_p++ = getBT1(next_info);
*out_p++ = getBT2(next_info);
continue;
case FOURbt:
*out_p++ = getBT0(next_info);
case THREEbt: /* fall through */
*out_p++ = getBT1(next_info);
*out_p++ = getBT2(next_info);
*out_p++ = getBT3(next_info);
continue;
case ILLEGAL:
goto illegal;
case UNDEF:
/* todo: add code for alternative behaviors */
rb_raise(rb_eRuntimeError /*@@@change exception*/, "conversion undefined for byte sequence");
continue;
}
continue;
illegal:
/* deal with illegal byte sequence */
/* todo: add code for alternative behaviors */
rb_raise(rb_eRuntimeError /*change exception*/, "illegal byte sequence");
continue;
}
/* cleanup */
*in_pos = in_p;
*out_pos = out_p;
}
/*
* String-specific code
*/
static VALUE
str_transcode(int argc, VALUE *argv, VALUE str, int bang)
{
VALUE dest;
long blen, slen, len;
char *buf, *bp, *sp, *fromp;
int tainted = 0;
rb_encoding *from_enc, *to_enc;
char *from_e, *to_e;
transcoder *my_transcoder;
int idx;
transcoding my_transcoding;
if (argc<1 || argc>2) {
rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
}
to_enc = NULL; /* todo: work out later, 'to' parameter may be Encoding,
or we want an encoding to set on result */
to_e = RSTRING_PTR(StringValue(argv[0]));
if (argc==1) {
from_enc = rb_enc_get(str);
from_e = (char *)rb_enc_name(from_enc);
}
else {
from_enc = NULL; /* todo: work out later, 'from' parameter may be Encoding */
from_e = RSTRING_PTR(StringValue(argv[1]));
}
/* strcasecmp: hope we are in C locale or locale-insensitive */
if (0==strcasecmp(from_e, to_e)) { /* TODO: add tests for US-ASCII-clean data and ASCII-compatible encodings */
if (bang) return str;
return rb_str_dup(str);
}
if (!(my_transcoder = transcode_dispatch(from_e, to_e))) {
rb_raise(rb_eArgError, "transcoding not supported (from %s to %s)", from_e, to_e);
}
fromp = sp = RSTRING_PTR(str);
slen = RSTRING_LEN(str);
blen = slen + 30; /* len + margin */
dest = str_new(0, 0, blen);
bp = buf = RSTRING_PTR(dest);
my_transcoding.ruby_string_dest = dest;
rb_str_locktmp(dest);
/* for simple testing: */
transcode_loop((unsigned char **)&fromp, (unsigned char **)&bp,
(unsigned char*)(sp+slen), (unsigned char*)(bp+blen),
my_transcoder, &my_transcoding);
if (fromp != sp+slen) {
rb_raise(rb_eArgError, "not fully converted, %d bytes left", sp+slen-fromp);
}
buf = RSTRING_PTR(dest);
blen = RSTRING_LEN(dest);
*bp = '\0';
rb_str_unlocktmp(dest);
if (bang) {
if (str_independent(str) && !STR_EMBED_P(str)) {
free(RSTRING_PTR(str));
}
STR_SET_NOEMBED(str);
STR_UNSET_NOCAPA(str);
RSTRING(str)->as.heap.ptr = buf;
RSTRING(str)->as.heap.aux.capa = blen;
RSTRING(dest)->as.heap.ptr = 0;
RSTRING(dest)->as.heap.len = 0;
}
else {
RBASIC(dest)->klass = rb_obj_class(str);
OBJ_INFECT(dest, str);
str = dest;
}
STR_SET_LEN(str, bp - buf);
/* set encoding */ /* would like to have an easier way to do this */
if ((idx = rb_enc_find_index(to_e)) < 0) {
if ((idx = rb_enc_find_index("ASCII-8BIT")) < 0) {
rb_raise(rb_eArgError, "unknown encoding name: ASCII-8BIT");
}
}
rb_enc_associate(str, rb_enc_from_index(idx));
if (tainted) OBJ_TAINT(str); /* is this needed??? */
return str;
}
/*
* call-seq:
* str.encode!(encoding) => str
* str.encode!(to_encoding, from_encoding) => str
*
* With one argument, transcodes the contents of <i>str</i> from
* str.encoding to +encoding+.
* With two arguments, transcodes the contents of <i>str</i> from
* from_encoding to to_encoding.
* Returns the string even if no changes were made.
*/
static VALUE
rb_str_transcode_bang(int argc, VALUE *argv, VALUE str)
{
return str_transcode(argc, argv, str, 1);
}
/*
* call-seq:
* str.encode(encoding) => str
*
* With one argument, returns a copy of <i>str</i> transcoded
* to encoding +encoding+.
* With two arguments, returns a copy of <i>str</i> transcoded
* from from_encoding to to_encoding.
*/
static VALUE
rb_str_transcode(int argc, VALUE *argv, VALUE str)
{
return str_transcode(argc, argv, str, 0);
}
/* function to fool the optimizer (avoid inlining transcode_loop) */
void
transcode_fool_the_optimizer (void)
{
unsigned char **in_pos, **out_pos, *in_stop, *out_stop;
transcoder *my_transcoder;
transcoding *my_transcoding;
transcode_loop(in_pos, out_pos, in_stop, out_stop,
my_transcoder, my_transcoding);
}
void
Init_transcode(void)
{
init_transcoder_table();
rb_define_method(rb_cString, "encode", rb_str_transcode, -1);
rb_define_method(rb_cString, "encode!", rb_str_transcode_bang, -1);
}

40
transcode_data.h Normal file
View file

@ -0,0 +1,40 @@
typedef unsigned char base_element;
typedef const void * const info_element;
typedef struct byte_lookup {
const base_element *base;
const void * const * const info;
} BYTE_LOOKUP;
#ifdef TRANSCODE_DATA
/* data file needs to treat this as a pointer, to remove warnings */
#define PType (const void * const)
#else
/* in code, this is treated as just an integer */
#define PType (int)
#endif
#define NOMAP (PType 0x01) /* single byte direct map */
#define ONEbt (0x02) /* one byte payload */
#define TWObt (0x03) /* two bytes payload */
#define THREEbt (0x05) /* three bytes payload */
#define FOURbt (0x06) /* four bytes payload, UTF-8 only, macros start at getBT0 */
#define ILLEGAL (PType 0x07) /* illegal byte sequence */
#define UNDEF (PType 0x09) /* legal but undefined */
#define ZERObt (PType 0x0A) /* zero bytes of payload, i.e. remove */
#define output1(b1) ((void*)((((unsigned char)(b1))<<8)|ONEbt))
#define output2(b1,b2) ((void*)((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|TWObt))
#define output3(b1,b2,b3) ((void*)((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|THREEbt))
#define output4(b0,b1,b2,b3) ((void*)((((unsigned char)(b1))<< 8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt))
#define getBT1(a) (((a)>> 8)&0xFF)
#define getBT2(a) (((a)>>16)&0xFF)
#define getBT3(a) (((a)>>24)&0xFF)
#define getBT0(a) ((((a)>> 5)&0x07)|0xF0) /* for UTF-8 only!!! */
/* do we need these??? maybe not, can be done with simple tables */
#define ONETRAIL /* legal but undefined if one more trailing UTF-8 */
#define TWOTRAIL /* legal but undefined if two more trailing UTF-8 */
#define THREETRAIL /* legal but undefined if three more trailing UTF-8 */

3260
transcode_data_iso_8859.c Normal file
View file

File diff suppressed because it is too large Load diff