ruby--ruby/transcode.c

/**********************************************************************

  transcode.c -

  $Author$
  $Date$
  created at: Tue Oct 30 16:10:22 JST 2007

  Copyright (C) 2007 Martin Duerst

**********************************************************************/

#include "ruby/ruby.h"
#include "ruby/encoding.h"

#include "transcode_data.h"


VALUE rb_str_tmp_new(long);
VALUE rb_str_shared_replace(VALUE, VALUE);

/*
 *  Dispatch data and logic
 */

/* extern declarations, should use some include file here */
extern const BYTE_LOOKUP from_ISO_8859_1;
extern const BYTE_LOOKUP from_ISO_8859_2;
extern const BYTE_LOOKUP from_ISO_8859_3;
extern const BYTE_LOOKUP from_ISO_8859_4;
extern const BYTE_LOOKUP from_ISO_8859_5;
extern const BYTE_LOOKUP from_ISO_8859_6;
extern const BYTE_LOOKUP from_ISO_8859_7;
extern const BYTE_LOOKUP from_ISO_8859_8;
extern const BYTE_LOOKUP from_ISO_8859_9;
extern const BYTE_LOOKUP from_ISO_8859_10;
extern const BYTE_LOOKUP from_ISO_8859_11;
extern const BYTE_LOOKUP from_ISO_8859_13;
extern const BYTE_LOOKUP from_ISO_8859_14;
extern const BYTE_LOOKUP from_ISO_8859_15;

extern const BYTE_LOOKUP to_ISO_8859_1;
extern const BYTE_LOOKUP to_ISO_8859_2;
extern const BYTE_LOOKUP to_ISO_8859_3;
extern const BYTE_LOOKUP to_ISO_8859_4;
extern const BYTE_LOOKUP to_ISO_8859_5;
extern const BYTE_LOOKUP to_ISO_8859_6;
extern const BYTE_LOOKUP to_ISO_8859_7;
extern const BYTE_LOOKUP to_ISO_8859_8;
extern const BYTE_LOOKUP to_ISO_8859_9;
extern const BYTE_LOOKUP to_ISO_8859_10;
extern const BYTE_LOOKUP to_ISO_8859_11;
extern const BYTE_LOOKUP to_ISO_8859_13;
extern const BYTE_LOOKUP to_ISO_8859_14;
extern const BYTE_LOOKUP to_ISO_8859_15;


/* declarations probably need to go into separate header file, e.g. transcode.h */

/* static structure, one per supported encoding pair */
typedef struct {
    const char *from_encoding;
    const char *to_encoding;
    BYTE_LOOKUP *conv_tree_start;
    int max_output;
    int from_utf8;
} transcoder;

/* todo: dynamic structure, one per conversion (stream) */

/* in the future, add some mechanism for dynamically adding stuff here */
#define MAX_TRANSCODERS 29  /* todo: fix: this number has to be adjusted by hand */
static transcoder transcoder_table[MAX_TRANSCODERS];

/* not sure why it's not possible to do relocatable initializations */
/* maybe the code here can be removed (changed to simple initialization) */
/* if we move this to another file???? */
static void
register_transcoder(const char *from_e, const char *to_e,
    const BYTE_LOOKUP *tree_start, int max_output, int from_utf8)
{
    static int n = 0;
    if (n >= MAX_TRANSCODERS) {
        /* we are initializing, is it okay to use rb_raise here? */
        rb_raise(rb_eRuntimeError /*change exception*/, "not enough transcoder slots");
    }
    transcoder_table[n].from_encoding = from_e;
    transcoder_table[n].to_encoding = to_e;
    transcoder_table[n].conv_tree_start = (BYTE_LOOKUP *)tree_start;
    transcoder_table[n].max_output = max_output;
    transcoder_table[n].from_utf8 = from_utf8;

    n++;
}

static void
init_transcoder_table(void)
{
    register_transcoder("ISO-8859-1",  "UTF-8", &from_ISO_8859_1, 2, 0);
    register_transcoder("ISO-8859-2",  "UTF-8", &from_ISO_8859_2, 2, 0);
    register_transcoder("ISO-8859-3",  "UTF-8", &from_ISO_8859_3, 2, 0);
    register_transcoder("ISO-8859-4",  "UTF-8", &from_ISO_8859_4, 2, 0);
    register_transcoder("ISO-8859-5",  "UTF-8", &from_ISO_8859_5, 3, 0);
    register_transcoder("ISO-8859-6",  "UTF-8", &from_ISO_8859_6, 2, 0);
    register_transcoder("ISO-8859-7",  "UTF-8", &from_ISO_8859_7, 3, 0);
    register_transcoder("ISO-8859-8",  "UTF-8", &from_ISO_8859_8, 3, 0);
    register_transcoder("ISO-8859-9",  "UTF-8", &from_ISO_8859_9, 2, 0);
    register_transcoder("ISO-8859-10", "UTF-8", &from_ISO_8859_10, 3, 0);
    register_transcoder("ISO-8859-11", "UTF-8", &from_ISO_8859_11, 3, 0);
    register_transcoder("ISO-8859-13", "UTF-8", &from_ISO_8859_13, 3, 0);
    register_transcoder("ISO-8859-14", "UTF-8", &from_ISO_8859_14, 3, 0);
    register_transcoder("ISO-8859-15", "UTF-8", &from_ISO_8859_15, 3, 0);
    register_transcoder("UTF-8", "ISO-8859-1",  &to_ISO_8859_1, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-2",  &to_ISO_8859_2, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-3",  &to_ISO_8859_3, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-4",  &to_ISO_8859_4, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-5",  &to_ISO_8859_5, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-6",  &to_ISO_8859_6, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-7",  &to_ISO_8859_7, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-8",  &to_ISO_8859_8, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-9",  &to_ISO_8859_9, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-10", &to_ISO_8859_10, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-11", &to_ISO_8859_11, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-13", &to_ISO_8859_13, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-14", &to_ISO_8859_14, 1, 1);
    register_transcoder("UTF-8", "ISO-8859-15", &to_ISO_8859_15, 1, 1);
    register_transcoder(NULL, NULL, NULL, 0, 0);
}


static transcoder*
transcode_dispatch(const char* from_encoding, const char* to_encoding)
{
    transcoder *candidate = transcoder_table;
    
    for (candidate = transcoder_table; candidate->from_encoding; candidate++)
        if (0==strcasecmp(from_encoding, candidate->from_encoding)
            && 0==strcasecmp(to_encoding, candidate->to_encoding))
                break;
    /* in the future, add multistep transcoding logic here */
    return candidate->from_encoding ? candidate : NULL;
}

/* dynamic structure, one per conversion (similar to iconv_t) */
/* may carry conversion state (e.g. for iso-2022-jp) */
typedef struct transcoding {
    VALUE ruby_string_dest; /* the String used as the conversion destination,
                               or NULL if something else is being converted */
    char *(*flush_func)(struct transcoding*, int, int);
} transcoding;


/*
 *  Transcoding engine logic
 */
static void
transcode_loop(char **in_pos, char **out_pos,
	       char *in_stop, char *out_stop,
	       transcoder *my_transcoder,
	       transcoding *my_transcoding)
{
    char *in_p = *in_pos, *out_p = *out_pos;
    const BYTE_LOOKUP *conv_tree_start = my_transcoder->conv_tree_start;
    const BYTE_LOOKUP *next_table;
    unsigned int next_offset;
    VALUE next_info;
    unsigned char next_byte;
    int from_utf8 = my_transcoder->from_utf8;
    char *out_s = out_stop - my_transcoder->max_output + 1;
    while (in_p < in_stop) {
        next_table = conv_tree_start;
        if (out_p >= out_s) {
	    int len = (out_p - *out_pos);
	    int new_len = (len + my_transcoder->max_output) * 2;
	    *out_pos = (*my_transcoding->flush_func)(my_transcoding, len, new_len);
	    out_p = *out_pos + len;
	    out_s = *out_pos + new_len - my_transcoder->max_output;
        }
        next_byte = (unsigned char)*in_p++;
      follow_byte:
        next_offset = next_table->base[next_byte];
        next_info = (VALUE)next_table->info[next_offset];
        switch (next_info & 0x1F) {
	  case NOMAP:
	    *out_p++ = next_byte;
	    continue;
	  case 0x00: case 0x04: case 0x08: case 0x0C:
	  case 0x10: case 0x14: case 0x18: case 0x1C:
	    if (in_p >= in_stop) {
		/* todo: deal with the case of backtracking */
		/* todo: deal with incomplete input (streaming) */
		goto illegal;
	    }
	    next_byte = (unsigned char)*in_p++;
	    if (from_utf8) {
		if ((next_byte&0xC0) == 0x80)
		    next_byte -= 0x80;
		else
		    goto illegal;
	    }
	    next_table = next_table->info[next_offset];
	    goto follow_byte;
            /* maybe rewrite the following cases to use fallthrough???? */
	  case ZERObt: /* drop input */
	    continue;
	  case ONEbt:
	    *out_p++ = getBT1(next_info);
	    continue;
	  case TWObt:
	    *out_p++ = getBT1(next_info);
	    *out_p++ = getBT2(next_info);
	    continue;
	  case FOURbt:
	    *out_p++ = getBT0(next_info);
	  case THREEbt: /* fall through */
	    *out_p++ = getBT1(next_info);
	    *out_p++ = getBT2(next_info);
	    *out_p++ = getBT3(next_info);
	    continue;
	  case ILLEGAL:
	    goto illegal;
	  case UNDEF:
	    /* todo: add code for alternative behaviors */
	    rb_raise(rb_eRuntimeError /*@@@change exception*/, "conversion undefined for byte sequence");
	    continue;
        }
        continue;
      illegal:
        /* deal with illegal byte sequence */
        /* todo: add code for alternative behaviors */
        rb_raise(rb_eRuntimeError /*change exception*/, "illegal byte sequence");
        continue;
    }
    /* cleanup */
    *in_pos  = in_p;
    *out_pos = out_p;
}


/*
 *  String-specific code
 */

static char *
str_transcoding_resize(transcoding *my_transcoding, int len, int new_len)
{
    VALUE dest_string = my_transcoding->ruby_string_dest;
    rb_str_resize(dest_string, new_len);
    return RSTRING_PTR(dest_string);
}

static VALUE
str_transcode(int argc, VALUE *argv, VALUE str)
{
    VALUE dest;
    long blen, slen;
    char *buf, *bp, *sp, *fromp;
    rb_encoding *from_enc, *to_enc;
    const char *from_e, *to_e;
    int from_encidx, to_encidx;
    VALUE from_encval, to_encval;
    transcoder *my_transcoder;
    transcoding my_transcoding;

    if (argc<1 || argc>2) {
	rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
    }
    if ((to_encidx = rb_to_encoding_index(to_encval = argv[0])) < 0) {
	to_enc = 0;
	to_e = StringValueCStr(to_encval);
    }
    else {
	to_enc = rb_enc_from_index(to_encidx);
	to_e = rb_enc_name(to_enc);
    }
    if (argc==1) {
        from_encidx = rb_enc_get_index(str);
	from_enc = rb_enc_from_index(from_encidx);
	from_e = rb_enc_name(from_enc);
    }
    else if ((from_encidx = rb_to_encoding_index(from_encval = argv[1])) < 0) {
	from_enc = 0;
	from_e = StringValueCStr(from_encval);
    }
    else {
	from_enc = rb_enc_from_index(from_encidx);
	from_e = rb_enc_name(from_enc);
    }

    if (from_enc && from_enc == to_enc) {
	return Qnil;
    }
    if (from_enc && to_enc && rb_enc_asciicompat(from_enc) && rb_enc_asciicompat(to_enc)) {
	if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT)
	    return Qnil;
    }
    if (strcasecmp(from_e, to_e) == 0) {
	return Qnil;
    }
    if (!(my_transcoder = transcode_dispatch(from_e, to_e))) {
	rb_raise(rb_eArgError, "transcoding not supported (from %s to %s)", from_e, to_e);
    }

    fromp = sp = RSTRING_PTR(str);
    slen = RSTRING_LEN(str);
    blen = slen + 30; /* len + margin */
    dest = rb_str_tmp_new(blen);
    bp = RSTRING_PTR(dest);
    my_transcoding.ruby_string_dest = dest;
    my_transcoding.flush_func = str_transcoding_resize;

    /* for simple testing: */
    transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), my_transcoder, &my_transcoding);
    if (fromp != sp+slen) {
	rb_raise(rb_eArgError, "not fully converted, %d bytes left", sp+slen-fromp);
    }
    buf = RSTRING_PTR(dest);
    *bp = '\0';
    rb_str_set_len(dest, bp - buf);

    /* set encoding */
    if (!to_enc) {
	to_encidx = rb_enc_replicate(to_e, rb_default_encoding());
	to_enc = rb_enc_from_index(to_encidx);
    }
    rb_enc_associate(dest, to_enc);

    return dest;
}

/*
 *  call-seq:
 *     str.encode!(encoding)   => str
 *     str.encode!(to_encoding, from_encoding)   => str
 *
 *  With one argument, transcodes the contents of <i>str</i> from
 *  str.encoding to +encoding+.
 *  With two arguments, transcodes the contents of <i>str</i> from
 *  from_encoding to to_encoding.
 *  Returns the string even if no changes were made.
 */

static VALUE
rb_str_transcode_bang(int argc, VALUE *argv, VALUE str)
{
    VALUE newstr = str_transcode(argc, argv, str);
    if (NIL_P(newstr)) return str;
    rb_str_shared_replace(str, newstr);
    return str;
}

/*
 *  call-seq:
 *     str.encode(encoding)   => str
 *
 *  With one argument, returns a copy of <i>str</i> transcoded
 *  to encoding +encoding+.
 *  With two arguments, returns a copy of <i>str</i> transcoded
 *  from from_encoding to to_encoding.
 */

static VALUE
rb_str_transcode(int argc, VALUE *argv, VALUE str)
{
    VALUE newstr = str_transcode(argc, argv, str);
    if (NIL_P(newstr)) return rb_str_dup(str);
    RBASIC(newstr)->klass = rb_obj_class(str);
    OBJ_INFECT(newstr, str);
    return newstr;
}

void
Init_transcode(void)
{
    init_transcoder_table();
    rb_define_method(rb_cString, "encode", rb_str_transcode, -1);
    rb_define_method(rb_cString, "encode!", rb_str_transcode_bang, -1);
}