kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

Recommit of JSON; fix mixed declarations.

Browse source 
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@27501 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
naruse 2010-04-26 06:27:27 +00:00
parent 27be3056dc
commit 3642494ce5
20 changed files with 2238 additions and 1766 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ChangeLog
View file

@ -1,7 +1,3 @@
Mon Apr 26 13:33:39 2010 NAKAMURA Usaku <usa@ruby-lang.org>
* ext/jason: came again after canceling gcc-ism.
Mon Apr 26 13:11:57 2010 Nobuyoshi Nakada <nobu@ruby-lang.org>
* parse.y (ripper_get_value): escape Qundef.

9
ext/json/ext/generator/extconf.rb
View file

@ -1,9 +0,0 @@
require 'mkmf'
require 'rbconfig'
if CONFIG['GCC'] == 'yes'
$CFLAGS += ' -Wall'
#$CFLAGS += ' -O0 -ggdb'
end
create_makefile 'json/ext/generator'

919
ext/json/ext/generator/generator.c
View file

@ -1,919 +0,0 @@
#include <string.h>
#include "ruby.h"
#if HAVE_RUBY_ST_H
#include "ruby/st.h"
#endif
#if HAVE_ST_H
#include "st.h"
#endif
#include "unicode.h"
#include <math.h>
#ifndef RHASH_TBL
#define RHASH_TBL(hsh) (RHASH(hsh)->tbl)
#endif
#ifndef RHASH_SIZE
#define RHASH_SIZE(hsh) (RHASH(hsh)->tbl->num_entries)
#endif
#ifndef RFLOAT_VALUE
#define RFLOAT_VALUE(val) (RFLOAT(val)->value)
#endif
#ifdef HAVE_RUBY_ENCODING_H
#include "ruby/encoding.h"
#define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
#else
#define FORCE_UTF8(obj)
#endif
#define check_max_nesting(state, depth) do { \
long current_nesting = 1 + depth; \
if (state->max_nesting != 0 && current_nesting > state->max_nesting) \
rb_raise(eNestingError, "nesting of %ld is too deep", current_nesting); \
} while (0);
static VALUE mJSON, mExt, mGenerator, cState, mGeneratorMethods, mObject,
mHash, mArray, mInteger, mFloat, mString, mString_Extend,
mTrueClass, mFalseClass, mNilClass, eGeneratorError,
eCircularDatastructure, eNestingError;
static ID i_to_s, i_to_json, i_new, i_indent, i_space, i_space_before,
i_object_nl, i_array_nl, i_check_circular, i_max_nesting,
i_allow_nan, i_pack, i_unpack, i_create_id, i_extend;
typedef struct JSON_Generator_StateStruct {
VALUE indent;
VALUE space;
VALUE space_before;
VALUE object_nl;
VALUE array_nl;
int check_circular;
VALUE seen;
VALUE memo;
VALUE depth;
long max_nesting;
int flag;
int allow_nan;
} JSON_Generator_State;
#define GET_STATE(self) \
JSON_Generator_State *state; \
Data_Get_Struct(self, JSON_Generator_State, state);
/*
* Document-module: JSON::Ext::Generator
*
* This is the JSON generator implemented as a C extension. It can be
* configured to be used by setting
*
* JSON.generator = JSON::Ext::Generator
*
* with the method generator= in JSON.
*
*/
static int hash_to_json_state_i(VALUE key, VALUE value, VALUE Vstate)
{
VALUE json, buf, Vdepth;
GET_STATE(Vstate);
buf = state->memo;
Vdepth = state->depth;
if (key == Qundef) return ST_CONTINUE;
if (state->flag) {
state->flag = 0;
rb_str_buf_cat2(buf, ",");
if (RSTRING_LEN(state->object_nl)) rb_str_buf_append(buf, state->object_nl);
}
if (RSTRING_LEN(state->object_nl)) {
rb_str_buf_append(buf, rb_str_times(state->indent, Vdepth));
}
json = rb_funcall(rb_funcall(key, i_to_s, 0), i_to_json, 2, Vstate, Vdepth);
Check_Type(json, T_STRING);
rb_str_buf_append(buf, json);
OBJ_INFECT(buf, json);
if (RSTRING_LEN(state->space_before)) {
rb_str_buf_append(buf, state->space_before);
}
rb_str_buf_cat2(buf, ":");
if (RSTRING_LEN(state->space)) rb_str_buf_append(buf, state->space);
json = rb_funcall(value, i_to_json, 2, Vstate, Vdepth);
Check_Type(json, T_STRING);
state->flag = 1;
rb_str_buf_append(buf, json);
OBJ_INFECT(buf, json);
state->depth = Vdepth;
state->memo = buf;
return ST_CONTINUE;
}
inline static VALUE mHash_json_transfrom(VALUE self, VALUE Vstate, VALUE Vdepth) {
long depth, len = RHASH_SIZE(self);
VALUE result;
GET_STATE(Vstate);
depth = 1 + FIX2LONG(Vdepth);
result = rb_str_buf_new(len);
state->memo = result;
state->depth = LONG2FIX(depth);
state->flag = 0;
rb_str_buf_cat2(result, "{");
if (RSTRING_LEN(state->object_nl)) rb_str_buf_append(result, state->object_nl);
rb_hash_foreach(self, hash_to_json_state_i, Vstate);
if (RSTRING_LEN(state->object_nl)) rb_str_buf_append(result, state->object_nl);
if (RSTRING_LEN(state->object_nl)) {
rb_str_buf_append(result, rb_str_times(state->indent, Vdepth));
}
rb_str_buf_cat2(result, "}");
return result;
}
static int hash_to_json_i(VALUE key, VALUE value, VALUE buf)
{
VALUE tmp;
if (key == Qundef) return ST_CONTINUE;
if (RSTRING_LEN(buf) > 1) rb_str_buf_cat2(buf, ",");
tmp = rb_funcall(rb_funcall(key, i_to_s, 0), i_to_json, 0);
Check_Type(tmp, T_STRING);
rb_str_buf_append(buf, tmp);
OBJ_INFECT(buf, tmp);
rb_str_buf_cat2(buf, ":");
tmp = rb_funcall(value, i_to_json, 0);
Check_Type(tmp, T_STRING);
rb_str_buf_append(buf, tmp);
OBJ_INFECT(buf, tmp);
return ST_CONTINUE;
}
/*
* call-seq: to_json(state = nil, depth = 0)
*
* Returns a JSON string containing a JSON object, that is unparsed from
* this Hash instance.
* _state_ is a JSON::State object, that can also be used to configure the
* produced JSON string output further.
* _depth_ is used to find out nesting depth, to indent accordingly.
*/
static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE Vstate, Vdepth, result;
long depth;
rb_scan_args(argc, argv, "02", &Vstate, &Vdepth);
depth = NIL_P(Vdepth) ? 0 : FIX2LONG(Vdepth);
if (NIL_P(Vstate)) {
long len = RHASH_SIZE(self);
result = rb_str_buf_new(len);
rb_str_buf_cat2(result, "{");
rb_hash_foreach(self, hash_to_json_i, result);
rb_str_buf_cat2(result, "}");
} else {
GET_STATE(Vstate);
check_max_nesting(state, depth);
if (state->check_circular) {
VALUE self_id = rb_obj_id(self);
if (RTEST(rb_hash_aref(state->seen, self_id))) {
rb_raise(eCircularDatastructure,
"circular data structures not supported!");
}
rb_hash_aset(state->seen, self_id, Qtrue);
result = mHash_json_transfrom(self, Vstate, LONG2FIX(depth));
rb_hash_delete(state->seen, self_id);
} else {
result = mHash_json_transfrom(self, Vstate, LONG2FIX(depth));
}
}
OBJ_INFECT(result, self);
FORCE_UTF8(result);
return result;
}
inline static VALUE mArray_json_transfrom(VALUE self, VALUE Vstate, VALUE Vdepth) {
long i, len = RARRAY_LEN(self);
VALUE shift, result;
long depth = NIL_P(Vdepth) ? 0 : FIX2LONG(Vdepth);
VALUE delim = rb_str_new2(",");
GET_STATE(Vstate);
check_max_nesting(state, depth);
if (state->check_circular) {
VALUE self_id = rb_obj_id(self);
rb_hash_aset(state->seen, self_id, Qtrue);
result = rb_str_buf_new(len);
if (RSTRING_LEN(state->array_nl)) rb_str_append(delim, state->array_nl);
shift = rb_str_times(state->indent, LONG2FIX(depth + 1));
rb_str_buf_cat2(result, "[");
OBJ_INFECT(result, self);
rb_str_buf_append(result, state->array_nl);
for (i = 0; i < len; i++) {
VALUE element = RARRAY_PTR(self)[i];
if (RTEST(rb_hash_aref(state->seen, rb_obj_id(element)))) {
rb_raise(eCircularDatastructure,
"circular data structures not supported!");
}
OBJ_INFECT(result, element);
if (i > 0) rb_str_buf_append(result, delim);
rb_str_buf_append(result, shift);
element = rb_funcall(element, i_to_json, 2, Vstate, LONG2FIX(depth + 1));
Check_Type(element, T_STRING);
rb_str_buf_append(result, element);
}
if (RSTRING_LEN(state->array_nl)) {
rb_str_buf_append(result, state->array_nl);
rb_str_buf_append(result, rb_str_times(state->indent, LONG2FIX(depth)));
}
rb_str_buf_cat2(result, "]");
rb_hash_delete(state->seen, self_id);
} else {
result = rb_str_buf_new(len);
OBJ_INFECT(result, self);
if (RSTRING_LEN(state->array_nl)) rb_str_append(delim, state->array_nl);
shift = rb_str_times(state->indent, LONG2FIX(depth + 1));
rb_str_buf_cat2(result, "[");
rb_str_buf_append(result, state->array_nl);
for (i = 0; i < len; i++) {
VALUE element = RARRAY_PTR(self)[i];
OBJ_INFECT(result, element);
if (i > 0) rb_str_buf_append(result, delim);
rb_str_buf_append(result, shift);
element = rb_funcall(element, i_to_json, 2, Vstate, LONG2FIX(depth + 1));
Check_Type(element, T_STRING);
rb_str_buf_append(result, element);
}
rb_str_buf_append(result, state->array_nl);
if (RSTRING_LEN(state->array_nl)) {
rb_str_buf_append(result, rb_str_times(state->indent, LONG2FIX(depth)));
}
rb_str_buf_cat2(result, "]");
}
return result;
}
/*
* call-seq: to_json(state = nil, depth = 0)
*
* Returns a JSON string containing a JSON array, that is unparsed from
* this Array instance.
* _state_ is a JSON::State object, that can also be used to configure the
* produced JSON string output further.
* _depth_ is used to find out nesting depth, to indent accordingly.
*/
static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) {
VALUE Vstate, Vdepth, result;
rb_scan_args(argc, argv, "02", &Vstate, &Vdepth);
if (NIL_P(Vstate)) {
long i, len = RARRAY_LEN(self);
result = rb_str_buf_new(2 + 2 * len);
rb_str_buf_cat2(result, "[");
OBJ_INFECT(result, self);
for (i = 0; i < len; i++) {
VALUE element = RARRAY_PTR(self)[i];
OBJ_INFECT(result, element);
if (i > 0) rb_str_buf_cat2(result, ",");
element = rb_funcall(element, i_to_json, 0);
Check_Type(element, T_STRING);
rb_str_buf_append(result, element);
}
rb_str_buf_cat2(result, "]");
} else {
result = mArray_json_transfrom(self, Vstate, Vdepth);
}
OBJ_INFECT(result, self);
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
* Returns a JSON string representation for this Integer number.
*/
static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result = rb_funcall(self, i_to_s, 0);
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
* Returns a JSON string representation for this Float number.
*/
static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self)
{
JSON_Generator_State *state = NULL;
VALUE Vstate, rest, tmp, result;
double value = RFLOAT_VALUE(self);
rb_scan_args(argc, argv, "01*", &Vstate, &rest);
if (!NIL_P(Vstate)) Data_Get_Struct(Vstate, JSON_Generator_State, state);
if (isinf(value)) {
if (!state || state->allow_nan) {
result = rb_funcall(self, i_to_s, 0);
} else {
tmp = rb_funcall(self, i_to_s, 0);
rb_raise(eGeneratorError, "%u: %s not allowed in JSON", __LINE__, StringValueCStr(tmp));
}
} else if (isnan(value)) {
if (!state || state->allow_nan) {
result = rb_funcall(self, i_to_s, 0);
} else {
tmp = rb_funcall(self, i_to_s, 0);
rb_raise(eGeneratorError, "%u: %s not allowed in JSON", __LINE__, StringValueCStr(tmp));
}
} else {
result = rb_funcall(self, i_to_s, 0);
}
FORCE_UTF8(result);
return result;
}
/*
* call-seq: String.included(modul)
*
* Extends _modul_ with the String::Extend module.
*/
static VALUE mString_included_s(VALUE self, VALUE modul) {
VALUE result = rb_funcall(modul, i_extend, 1, mString_Extend);
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
* This string should be encoded with UTF-8 A call to this method
* returns a JSON string encoded with UTF16 big endian characters as
* \u????.
*/
static VALUE mString_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result = rb_str_buf_new(RSTRING_LEN(self));
rb_str_buf_cat2(result, "\"");
JSON_convert_UTF8_to_JSON(result, self, strictConversion);
rb_str_buf_cat2(result, "\"");
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json_raw_object()
*
* This method creates a raw object hash, that can be nested into
* other data structures and will be unparsed as a raw string. This
* method should be used, if you want to convert raw strings to JSON
* instead of UTF-8 strings, e. g. binary data.
*/
static VALUE mString_to_json_raw_object(VALUE self) {
VALUE ary;
VALUE result = rb_hash_new();
rb_hash_aset(result, rb_funcall(mJSON, i_create_id, 0), rb_class_name(rb_obj_class(self)));
ary = rb_funcall(self, i_unpack, 1, rb_str_new2("C*"));
rb_hash_aset(result, rb_str_new2("raw"), ary);
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json_raw(*args)
*
* This method creates a JSON text from the result of a call to
* to_json_raw_object of this String.
*/
static VALUE mString_to_json_raw(int argc, VALUE *argv, VALUE self) {
VALUE result, obj = mString_to_json_raw_object(self);
Check_Type(obj, T_HASH);
result = mHash_to_json(argc, argv, obj);
FORCE_UTF8(result);
return result;
}
/*
* call-seq: json_create(o)
*
* Raw Strings are JSON Objects (the raw bytes are stored in an array for the
* key "raw"). The Ruby String can be created by this module method.
*/
static VALUE mString_Extend_json_create(VALUE self, VALUE o) {
VALUE ary;
Check_Type(o, T_HASH);
ary = rb_hash_aref(o, rb_str_new2("raw"));
return rb_funcall(ary, i_pack, 1, rb_str_new2("C*"));
}
/*
* call-seq: to_json(*)
*
* Returns a JSON string for true: 'true'.
*/
static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result = rb_str_new2("true");
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
* Returns a JSON string for false: 'false'.
*/
static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result = rb_str_new2("false");
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
*/
static VALUE mNilClass_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result = rb_str_new2("null");
FORCE_UTF8(result);
return result;
}
/*
* call-seq: to_json(*)
*
* Converts this object to a string (calling #to_s), converts
* it to a JSON string, and returns the result. This is a fallback, if no
* special method #to_json was defined for some object.
*/
static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self)
{
VALUE result, string = rb_funcall(self, i_to_s, 0);
Check_Type(string, T_STRING);
result = mString_to_json(argc, argv, string);
FORCE_UTF8(result);
return result;
}
/*
* Document-class: JSON::Ext::Generator::State
*
* This class is used to create State instances, that are use to hold data
* while generating a JSON text from a a Ruby data structure.
*/
static void State_mark(JSON_Generator_State *state)
{
rb_gc_mark_maybe(state->indent);
rb_gc_mark_maybe(state->space);
rb_gc_mark_maybe(state->space_before);
rb_gc_mark_maybe(state->object_nl);
rb_gc_mark_maybe(state->array_nl);
rb_gc_mark_maybe(state->seen);
rb_gc_mark_maybe(state->memo);
rb_gc_mark_maybe(state->depth);
}
static JSON_Generator_State *State_allocate()
{
JSON_Generator_State *state = ALLOC(JSON_Generator_State);
return state;
}
static VALUE cState_s_allocate(VALUE klass)
{
JSON_Generator_State *state = State_allocate();
return Data_Wrap_Struct(klass, State_mark, -1, state);
}
/*
* call-seq: configure(opts)
*
* Configure this State instance with the Hash _opts_, and return
* itself.
*/
static VALUE cState_configure(VALUE self, VALUE opts)
{
VALUE tmp;
GET_STATE(self);
tmp = rb_convert_type(opts, T_HASH, "Hash", "to_hash");
if (NIL_P(tmp)) tmp = rb_convert_type(opts, T_HASH, "Hash", "to_h");
if (NIL_P(tmp)) {
rb_raise(rb_eArgError, "opts has to be hash like or convertable into a hash");
}
opts = tmp;
tmp = rb_hash_aref(opts, ID2SYM(i_indent));
if (RTEST(tmp)) {
Check_Type(tmp, T_STRING);
state->indent = tmp;
}
tmp = rb_hash_aref(opts, ID2SYM(i_space));
if (RTEST(tmp)) {
Check_Type(tmp, T_STRING);
state->space = tmp;
}
tmp = rb_hash_aref(opts, ID2SYM(i_space_before));
if (RTEST(tmp)) {
Check_Type(tmp, T_STRING);
state->space_before = tmp;
}
tmp = rb_hash_aref(opts, ID2SYM(i_array_nl));
if (RTEST(tmp)) {
Check_Type(tmp, T_STRING);
state->array_nl = tmp;
}
tmp = rb_hash_aref(opts, ID2SYM(i_object_nl));
if (RTEST(tmp)) {
Check_Type(tmp, T_STRING);
state->object_nl = tmp;
}
tmp = ID2SYM(i_check_circular);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
tmp = rb_hash_aref(opts, ID2SYM(i_check_circular));
state->check_circular = RTEST(tmp);
} else {
state->check_circular = 1;
}
tmp = ID2SYM(i_max_nesting);
state->max_nesting = 19;
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
VALUE max_nesting = rb_hash_aref(opts, tmp);
if (RTEST(max_nesting)) {
Check_Type(max_nesting, T_FIXNUM);
state->max_nesting = FIX2LONG(max_nesting);
} else {
state->max_nesting = 0;
}
}
tmp = rb_hash_aref(opts, ID2SYM(i_allow_nan));
state->allow_nan = RTEST(tmp);
return self;
}
/*
* call-seq: to_h
*
* Returns the configuration instance variables as a hash, that can be
* passed to the configure method.
*/
static VALUE cState_to_h(VALUE self)
{
VALUE result = rb_hash_new();
GET_STATE(self);
rb_hash_aset(result, ID2SYM(i_indent), state->indent);
rb_hash_aset(result, ID2SYM(i_space), state->space);
rb_hash_aset(result, ID2SYM(i_space_before), state->space_before);
rb_hash_aset(result, ID2SYM(i_object_nl), state->object_nl);
rb_hash_aset(result, ID2SYM(i_array_nl), state->array_nl);
rb_hash_aset(result, ID2SYM(i_check_circular), state->check_circular ? Qtrue : Qfalse);
rb_hash_aset(result, ID2SYM(i_allow_nan), state->allow_nan ? Qtrue : Qfalse);
rb_hash_aset(result, ID2SYM(i_max_nesting), LONG2FIX(state->max_nesting));
return result;
}
/*
* call-seq: new(opts = {})
*
* Instantiates a new State object, configured by _opts_.
*
* _opts_ can have the following keys:
*
* * *indent*: a string used to indent levels (default: ''),
* * *space*: a string that is put after, a : or , delimiter (default: ''),
* * *space_before*: a string that is put before a : pair delimiter (default: ''),
* * *object_nl*: a string that is put at the end of a JSON object (default: ''),
* * *array_nl*: a string that is put at the end of a JSON array (default: ''),
* * *check_circular*: true if checking for circular data structures
* should be done, false (the default) otherwise.
* * *allow_nan*: true if NaN, Infinity, and -Infinity should be
* generated, otherwise an exception is thrown, if these values are
* encountered. This options defaults to false.
*/
static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
{
VALUE opts;
GET_STATE(self);
rb_scan_args(argc, argv, "01", &opts);
state->indent = rb_str_new2("");
state->space = rb_str_new2("");
state->space_before = rb_str_new2("");
state->array_nl = rb_str_new2("");
state->object_nl = rb_str_new2("");
if (NIL_P(opts)) {
state->check_circular = 1;
state->allow_nan = 0;
state->max_nesting = 19;
} else {
cState_configure(self, opts);
}
state->seen = rb_hash_new();
state->memo = Qnil;
state->depth = INT2FIX(0);
return self;
}
/*
* call-seq: from_state(opts)
*
* Creates a State object from _opts_, which ought to be Hash to create a
* new State instance configured by _opts_, something else to create an
* unconfigured instance. If _opts_ is a State object, it is just returned.
*/
static VALUE cState_from_state_s(VALUE self, VALUE opts)
{
if (rb_obj_is_kind_of(opts, self)) {
return opts;
} else if (rb_obj_is_kind_of(opts, rb_cHash)) {
return rb_funcall(self, i_new, 1, opts);
} else {
return rb_funcall(self, i_new, 0);
}
}
/*
* call-seq: indent()
*
* This string is used to indent levels in the JSON text.
*/
static VALUE cState_indent(VALUE self)
{
GET_STATE(self);
return state->indent;
}
/*
* call-seq: indent=(indent)
*
* This string is used to indent levels in the JSON text.
*/
static VALUE cState_indent_set(VALUE self, VALUE indent)
{
GET_STATE(self);
Check_Type(indent, T_STRING);
return state->indent = indent;
}
/*
* call-seq: space()
*
* This string is used to insert a space between the tokens in a JSON
* string.
*/
static VALUE cState_space(VALUE self)
{
GET_STATE(self);
return state->space;
}
/*
* call-seq: space=(space)
*
* This string is used to insert a space between the tokens in a JSON
* string.
*/
static VALUE cState_space_set(VALUE self, VALUE space)
{
GET_STATE(self);
Check_Type(space, T_STRING);
return state->space = space;
}
/*
* call-seq: space_before()
*
* This string is used to insert a space before the ':' in JSON objects.
*/
static VALUE cState_space_before(VALUE self)
{
GET_STATE(self);
return state->space_before;
}
/*
* call-seq: space_before=(space_before)
*
* This string is used to insert a space before the ':' in JSON objects.
*/
static VALUE cState_space_before_set(VALUE self, VALUE space_before)
{
GET_STATE(self);
Check_Type(space_before, T_STRING);
return state->space_before = space_before;
}
/*
* call-seq: object_nl()
*
* This string is put at the end of a line that holds a JSON object (or
* Hash).
*/
static VALUE cState_object_nl(VALUE self)
{
GET_STATE(self);
return state->object_nl;
}
/*
* call-seq: object_nl=(object_nl)
*
* This string is put at the end of a line that holds a JSON object (or
* Hash).
*/
static VALUE cState_object_nl_set(VALUE self, VALUE object_nl)
{
GET_STATE(self);
Check_Type(object_nl, T_STRING);
return state->object_nl = object_nl;
}
/*
* call-seq: array_nl()
*
* This string is put at the end of a line that holds a JSON array.
*/
static VALUE cState_array_nl(VALUE self)
{
GET_STATE(self);
return state->array_nl;
}
/*
* call-seq: array_nl=(array_nl)
*
* This string is put at the end of a line that holds a JSON array.
*/
static VALUE cState_array_nl_set(VALUE self, VALUE array_nl)
{
GET_STATE(self);
Check_Type(array_nl, T_STRING);
return state->array_nl = array_nl;
}
/*
* call-seq: check_circular?
*
* Returns true, if circular data structures should be checked,
* otherwise returns false.
*/
static VALUE cState_check_circular_p(VALUE self)
{
GET_STATE(self);
return state->check_circular ? Qtrue : Qfalse;
}
/*
* call-seq: max_nesting
*
* This integer returns the maximum level of data structure nesting in
* the generated JSON, max_nesting = 0 if no maximum is checked.
*/
static VALUE cState_max_nesting(VALUE self)
{
GET_STATE(self);
return LONG2FIX(state->max_nesting);
}
/*
* call-seq: max_nesting=(depth)
*
* This sets the maximum level of data structure nesting in the generated JSON
* to the integer depth, max_nesting = 0 if no maximum should be checked.
*/
static VALUE cState_max_nesting_set(VALUE self, VALUE depth)
{
GET_STATE(self);
Check_Type(depth, T_FIXNUM);
state->max_nesting = FIX2LONG(depth);
return Qnil;
}
/*
* call-seq: allow_nan?
*
* Returns true, if NaN, Infinity, and -Infinity should be generated, otherwise
* returns false.
*/
static VALUE cState_allow_nan_p(VALUE self)
{
GET_STATE(self);
return state->allow_nan ? Qtrue : Qfalse;
}
/*
* call-seq: seen?(object)
*
* Returns _true_, if _object_ was already seen during this generating run.
*/
static VALUE cState_seen_p(VALUE self, VALUE object)
{
GET_STATE(self);
return rb_hash_aref(state->seen, rb_obj_id(object));
}
/*
* call-seq: remember(object)
*
* Remember _object_, to find out if it was already encountered (if a cyclic
* data structure is rendered).
*/
static VALUE cState_remember(VALUE self, VALUE object)
{
GET_STATE(self);
return rb_hash_aset(state->seen, rb_obj_id(object), Qtrue);
}
/*
* call-seq: forget(object)
*
* Forget _object_ for this generating run.
*/
static VALUE cState_forget(VALUE self, VALUE object)
{
GET_STATE(self);
return rb_hash_delete(state->seen, rb_obj_id(object));
}
/*
*
*/
void Init_generator()
{
rb_require("json/common");
mJSON = rb_define_module("JSON");
mExt = rb_define_module_under(mJSON, "Ext");
mGenerator = rb_define_module_under(mExt, "Generator");
eGeneratorError = rb_path2class("JSON::GeneratorError");
eCircularDatastructure = rb_path2class("JSON::CircularDatastructure");
eNestingError = rb_path2class("JSON::NestingError");
cState = rb_define_class_under(mGenerator, "State", rb_cObject);
rb_define_alloc_func(cState, cState_s_allocate);
rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1);
rb_define_method(cState, "initialize", cState_initialize, -1);
rb_define_method(cState, "indent", cState_indent, 0);
rb_define_method(cState, "indent=", cState_indent_set, 1);
rb_define_method(cState, "space", cState_space, 0);
rb_define_method(cState, "space=", cState_space_set, 1);
rb_define_method(cState, "space_before", cState_space_before, 0);
rb_define_method(cState, "space_before=", cState_space_before_set, 1);
rb_define_method(cState, "object_nl", cState_object_nl, 0);
rb_define_method(cState, "object_nl=", cState_object_nl_set, 1);
rb_define_method(cState, "array_nl", cState_array_nl, 0);
rb_define_method(cState, "array_nl=", cState_array_nl_set, 1);
rb_define_method(cState, "check_circular?", cState_check_circular_p, 0);
rb_define_method(cState, "max_nesting", cState_max_nesting, 0);
rb_define_method(cState, "max_nesting=", cState_max_nesting_set, 1);
rb_define_method(cState, "allow_nan?", cState_allow_nan_p, 0);
rb_define_method(cState, "seen?", cState_seen_p, 1);
rb_define_method(cState, "remember", cState_remember, 1);
rb_define_method(cState, "forget", cState_forget, 1);
rb_define_method(cState, "configure", cState_configure, 1);
rb_define_method(cState, "to_h", cState_to_h, 0);
mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
mObject = rb_define_module_under(mGeneratorMethods, "Object");
rb_define_method(mObject, "to_json", mObject_to_json, -1);
mHash = rb_define_module_under(mGeneratorMethods, "Hash");
rb_define_method(mHash, "to_json", mHash_to_json, -1);
mArray = rb_define_module_under(mGeneratorMethods, "Array");
rb_define_method(mArray, "to_json", mArray_to_json, -1);
mInteger = rb_define_module_under(mGeneratorMethods, "Integer");
rb_define_method(mInteger, "to_json", mInteger_to_json, -1);
mFloat = rb_define_module_under(mGeneratorMethods, "Float");
rb_define_method(mFloat, "to_json", mFloat_to_json, -1);
mString = rb_define_module_under(mGeneratorMethods, "String");
rb_define_singleton_method(mString, "included", mString_included_s, 1);
rb_define_method(mString, "to_json", mString_to_json, -1);
rb_define_method(mString, "to_json_raw", mString_to_json_raw, -1);
rb_define_method(mString, "to_json_raw_object", mString_to_json_raw_object, 0);
mString_Extend = rb_define_module_under(mString, "Extend");
rb_define_method(mString_Extend, "json_create", mString_Extend_json_create, 1);
mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass");
rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1);
mFalseClass = rb_define_module_under(mGeneratorMethods, "FalseClass");
rb_define_method(mFalseClass, "to_json", mFalseClass_to_json, -1);
mNilClass = rb_define_module_under(mGeneratorMethods, "NilClass");
rb_define_method(mNilClass, "to_json", mNilClass_to_json, -1);
i_to_s = rb_intern("to_s");
i_to_json = rb_intern("to_json");
i_new = rb_intern("new");
i_indent = rb_intern("indent");
i_space = rb_intern("space");
i_space_before = rb_intern("space_before");
i_object_nl = rb_intern("object_nl");
i_array_nl = rb_intern("array_nl");
i_check_circular = rb_intern("check_circular");
i_max_nesting = rb_intern("max_nesting");
i_allow_nan = rb_intern("allow_nan");
i_pack = rb_intern("pack");
i_unpack = rb_intern("unpack");
i_create_id = rb_intern("create_id");
i_extend = rb_intern("extend");
}

180
ext/json/ext/generator/unicode.c
View file

@ -1,180 +0,0 @@
#include "unicode.h"
#define unicode_escape(buffer, character) \
snprintf(buf, 7, "\\u%04x", (unsigned int) (character)); \
rb_str_buf_cat(buffer, buf, 6);
/*
* Copyright 2001-2004 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/
/*
* Index into the table below with the first byte of a UTF-8 sequence to
* get the number of trailing bytes that are supposed to follow it.
* Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
* left as-is for anyone who may want to do such conversion, which was
* allowed in earlier algorithms.
*/
static const char trailingBytesForUTF8[256] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};
/*
* Magic values subtracted from a buffer value during UTF8 conversion.
* This table contains as many values as there might be trailing bytes
* in a UTF-8 sequence.
*/
static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
0x03C82080UL, 0xFA082080UL, 0x82082080UL };
/*
* Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
* into the first byte, depending on how many bytes follow. There are
* as many entries in this table as there are UTF-8 sequence types.
* (I.e., one byte sequence, two byte... etc.). Remember that sequencs
* for *legal* UTF-8 will be 4 or fewer bytes total.
*/
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
/*
* Utility routine to tell whether a sequence of bytes is legal UTF-8.
* This must be called with the length pre-determined by the first byte.
* If not calling this from ConvertUTF8to*, then the length can be set by:
* length = trailingBytesForUTF8[*source]+1;
* and the sequence is illegal right away if there aren't that many bytes
* available.
* If presented with a length > 4, this returns 0. The Unicode
* definition of UTF-8 goes up to 4-byte sequences.
*/
inline static unsigned char isLegalUTF8(const UTF8 *source, int length)
{
UTF8 a;
const UTF8 *srcptr = source+length;
switch (length) {
default: return 0;
/* Everything else falls through when "1"... */
case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
case 2: if ((a = (*--srcptr)) > 0xBF) return 0;
switch (*source) {
/* no fall-through in this inner switch */
case 0xE0: if (a < 0xA0) return 0; break;
case 0xED: if (a > 0x9F) return 0; break;
case 0xF0: if (a < 0x90) return 0; break;
case 0xF4: if (a > 0x8F) return 0; break;
default: if (a < 0x80) return 0;
}
case 1: if (*source >= 0x80 && *source < 0xC2) return 0;
}
if (*source > 0xF4) return 0;
return 1;
}
void JSON_convert_UTF8_to_JSON(VALUE buffer, VALUE string, ConversionFlags flags)
{
char buf[7];
const UTF8* source = (UTF8 *) RSTRING_PTR(string);
const UTF8* sourceEnd = source + RSTRING_LEN(string);
while (source < sourceEnd) {
UTF32 ch = 0;
unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
if (source + extraBytesToRead >= sourceEnd) {
rb_raise(rb_path2class("JSON::GeneratorError"),
"partial character in source, but hit end");
}
if (!isLegalUTF8(source, extraBytesToRead+1)) {
rb_raise(rb_path2class("JSON::GeneratorError"),
"source sequence is illegal/malformed");
}
/*
* The cases all fall through. See "Note A" below.
*/
switch (extraBytesToRead) {
case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
case 3: ch += *source++; ch <<= 6;
case 2: ch += *source++; ch <<= 6;
case 1: ch += *source++; ch <<= 6;
case 0: ch += *source++;
}
ch -= offsetsFromUTF8[extraBytesToRead];
if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
/* UTF-16 surrogate values are illegal in UTF-32 */
if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
if (flags == strictConversion) {
source -= (extraBytesToRead+1); /* return to the illegal value itself */
rb_raise(rb_path2class("JSON::GeneratorError"),
"source sequence is illegal/malformed");
} else {
unicode_escape(buffer, UNI_REPLACEMENT_CHAR);
}
} else {
/* normal case */
if (ch == '"') {
rb_str_buf_cat2(buffer, "\\\"");
} else if (ch == '\\') {
rb_str_buf_cat2(buffer, "\\\\");
} else if (ch >= 0x20 && ch <= 0x7f) {
rb_str_buf_cat(buffer, (char *) source - 1, 1);
} else if (ch == '\n') {
rb_str_buf_cat2(buffer, "\\n");
} else if (ch == '\r') {
rb_str_buf_cat2(buffer, "\\r");
} else if (ch == '\t') {
rb_str_buf_cat2(buffer, "\\t");
} else if (ch == '\f') {
rb_str_buf_cat2(buffer, "\\f");
} else if (ch == '\b') {
rb_str_buf_cat2(buffer, "\\b");
} else if (ch < 0x20) {
unicode_escape(buffer, (UTF16) ch);
} else {
unicode_escape(buffer, (UTF16) ch);
}
}
} else if (ch > UNI_MAX_UTF16) {
if (flags == strictConversion) {
source -= (extraBytesToRead+1); /* return to the start */
rb_raise(rb_path2class("JSON::GeneratorError"),
"source sequence is illegal/malformed");
} else {
unicode_escape(buffer, UNI_REPLACEMENT_CHAR);
}
} else {
/* target is a character in range 0xFFFF - 0x10FFFF. */
ch -= halfBase;
unicode_escape(buffer, (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START));
unicode_escape(buffer, (UTF16)((ch & halfMask) + UNI_SUR_LOW_START));
}
}
}

53
ext/json/ext/generator/unicode.h
View file

@ -1,53 +0,0 @@
#include "ruby.h"
#ifndef _GENERATOR_UNICODE_H_
#define _GENERATOR_UNICODE_H_
typedef enum {
conversionOK = 0, /* conversion successful */
sourceExhausted, /* partial character in source, but hit end */
targetExhausted, /* insuff. room in target for conversion */
sourceIllegal /* source sequence is illegal/malformed */
} ConversionResult;
typedef enum {
strictConversion = 0,
lenientConversion
} ConversionFlags;
typedef unsigned long UTF32; /* at least 32 bits */
typedef unsigned short UTF16; /* at least 16 bits */
typedef unsigned char UTF8; /* typically 8 bits */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
#define UNI_SUR_HIGH_START (UTF32)0xD800
#define UNI_SUR_HIGH_END (UTF32)0xDBFF
#define UNI_SUR_LOW_START (UTF32)0xDC00
#define UNI_SUR_LOW_END (UTF32)0xDFFF
static const int halfShift = 10; /* used for shifting by 10 bits */
static const UTF32 halfBase = 0x0010000UL;
static const UTF32 halfMask = 0x3FFUL;
void JSON_convert_UTF8_to_JSON(VALUE buffer, VALUE string, ConversionFlags flags);
#ifndef RARRAY_PTR
#define RARRAY_PTR(ARRAY) RARRAY(ARRAY)->ptr
#endif
#ifndef RARRAY_LEN
#define RARRAY_LEN(ARRAY) RARRAY(ARRAY)->len
#endif
#ifndef RSTRING_PTR
#define RSTRING_PTR(string) RSTRING(string)->ptr
#endif
#ifndef RSTRING_LEN
#define RSTRING_LEN(string) RSTRING(string)->len
#endif
#endif

9
ext/json/ext/parser/extconf.rb
View file

@ -1,9 +0,0 @@
require 'mkmf'
require 'rbconfig'
if CONFIG['GCC'] == 'yes'
$CFLAGS += ' -Wall'
#$CFLAGS += ' -O0 -ggdb'
end
create_makefile 'json/ext/parser'

154
ext/json/ext/parser/unicode.c
View file

@ -1,154 +0,0 @@
#include "unicode.h"
/*
* Copyright 2001-2004 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/
/*
* Index into the table below with the first byte of a UTF-8 sequence to
* get the number of trailing bytes that are supposed to follow it.
* Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
* left as-is for anyone who may want to do such conversion, which was
* allowed in earlier algorithms.
*/
static const char trailingBytesForUTF8[256] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};
/*
* Magic values subtracted from a buffer value during UTF8 conversion.
* This table contains as many values as there might be trailing bytes
* in a UTF-8 sequence.
*/
static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
0x03C82080UL, 0xFA082080UL, 0x82082080UL };
/*
* Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
* into the first byte, depending on how many bytes follow. There are
* as many entries in this table as there are UTF-8 sequence types.
* (I.e., one byte sequence, two byte... etc.). Remember that sequencs
* for *legal* UTF-8 will be 4 or fewer bytes total.
*/
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
char *JSON_convert_UTF16_to_UTF8 (
VALUE buffer,
char *source,
char *sourceEnd,
ConversionFlags flags)
{
UTF16 *tmp, *tmpPtr, *tmpEnd;
char buf[5];
long n = 0, i;
char *p = source - 1;
while (p < sourceEnd && p[0] == '\\' && p[1] == 'u') {
p += 6;
n++;
}
p = source + 1;
buf[4] = 0;
tmpPtr = tmp = ALLOC_N(UTF16, n);
tmpEnd = tmp + n;
for (i = 0; i < n; i++) {
buf[0] = *p++;
buf[1] = *p++;
buf[2] = *p++;
buf[3] = *p++;
tmpPtr[i] = (UTF16)strtol(buf, NULL, 16);
p += 2;
}
while (tmpPtr < tmpEnd) {
UTF32 ch;
unsigned short bytesToWrite = 0;
const UTF32 byteMask = 0xBF;
const UTF32 byteMark = 0x80;
ch = *tmpPtr++;
/* If we have a surrogate pair, convert to UTF32 first. */
if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
/* If the 16 bits following the high surrogate are in the source
* buffer... */
if (tmpPtr < tmpEnd) {
UTF32 ch2 = *tmpPtr;
/* If it's a low surrogate, convert to UTF32. */
if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
+ (ch2 - UNI_SUR_LOW_START) + halfBase;
++tmpPtr;
} else if (flags == strictConversion) { /* it's an unpaired high surrogate */
ruby_xfree(tmp);
rb_raise(rb_path2class("JSON::ParserError"),
"source sequence is illegal/malformed near %s", source);
}
} else { /* We don't have the 16 bits following the high surrogate. */
ruby_xfree(tmp);
rb_raise(rb_path2class("JSON::ParserError"),
"partial character in source, but hit end near %s", source);
break;
}
} else if (flags == strictConversion) {
/* UTF-16 surrogate values are illegal in UTF-32 */
if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
ruby_xfree(tmp);
rb_raise(rb_path2class("JSON::ParserError"),
"source sequence is illegal/malformed near %s", source);
}
}
/* Figure out how many bytes the result will require */
if (ch < (UTF32) 0x80) {
bytesToWrite = 1;
} else if (ch < (UTF32) 0x800) {
bytesToWrite = 2;
} else if (ch < (UTF32) 0x10000) {
bytesToWrite = 3;
} else if (ch < (UTF32) 0x110000) {
bytesToWrite = 4;
} else {
bytesToWrite = 3;
ch = UNI_REPLACEMENT_CHAR;
}
buf[0] = 0;
buf[1] = 0;
buf[2] = 0;
buf[3] = 0;
p = buf + bytesToWrite;
switch (bytesToWrite) { /* note: everything falls through. */
case 4: *--p = (UTF8) ((ch | byteMark) & byteMask); ch >>= 6;
case 3: *--p = (UTF8) ((ch | byteMark) & byteMask); ch >>= 6;
case 2: *--p = (UTF8) ((ch | byteMark) & byteMask); ch >>= 6;
case 1: *--p = (UTF8) (ch | firstByteMark[bytesToWrite]);
}
rb_str_buf_cat(buffer, p, bytesToWrite);
}
ruby_xfree(tmp);
source += 5 + (n - 1) * 6;
return source;
}

58
ext/json/ext/parser/unicode.h
View file

@ -1,58 +0,0 @@
#ifndef _PARSER_UNICODE_H_
#define _PARSER_UNICODE_H_
#include "ruby.h"
typedef unsigned long UTF32; /* at least 32 bits */
typedef unsigned short UTF16; /* at least 16 bits */
typedef unsigned char UTF8; /* typically 8 bits */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
#define UNI_SUR_HIGH_START (UTF32)0xD800
#define UNI_SUR_HIGH_END (UTF32)0xDBFF
#define UNI_SUR_LOW_START (UTF32)0xDC00
#define UNI_SUR_LOW_END (UTF32)0xDFFF
static const int halfShift = 10; /* used for shifting by 10 bits */
static const UTF32 halfBase = 0x0010000UL;
static const UTF32 halfMask = 0x3FFUL;
typedef enum {
conversionOK = 0, /* conversion successful */
sourceExhausted, /* partial character in source, but hit end */
targetExhausted, /* insuff. room in target for conversion */
sourceIllegal /* source sequence is illegal/malformed */
} ConversionResult;
typedef enum {
strictConversion = 0,
lenientConversion
} ConversionFlags;
char *JSON_convert_UTF16_to_UTF8 (
VALUE buffer,
char *source,
char *sourceEnd,
ConversionFlags flags);
#ifndef RARRAY_PTR
#define RARRAY_PTR(ARRAY) RARRAY(ARRAY)->ptr
#endif
#ifndef RARRAY_LEN
#define RARRAY_LEN(ARRAY) RARRAY(ARRAY)->len
#endif
#ifndef RSTRING_PTR
#define RSTRING_PTR(string) RSTRING(string)->ptr
#endif
#ifndef RSTRING_LEN
#define RSTRING_LEN(string) RSTRING(string)->len
#endif
#endif

4
ext/json/generator/extconf.rb Normal file
View file

@ -0,0 +1,4 @@
require 'mkmf'
require 'rbconfig'
create_makefile 'json/ext/generator'

1341
ext/json/generator/generator.c Normal file
View file

File diff suppressed because it is too large Load diff

170
ext/json/generator/generator.h Normal file
View file

@ -0,0 +1,170 @@
#ifndef _GENERATOR_H_
#define _GENERATOR_H_
#include <string.h>
#include <assert.h>
#include <math.h>
#include "ruby.h"
#if HAVE_RUBY_RE_H
#include "ruby/re.h"
#endif
#if HAVE_RE_H
#include "re.h"
#endif
#ifdef HAVE_RUBY_ENCODING_H
#include "ruby/encoding.h"
#define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
#else
#define FORCE_UTF8(obj)
#endif
#define option_given_p(opts, key) RTEST(rb_funcall(opts, i_key_p, 1, key))
#ifndef RHASH_SIZE
#define RHASH_SIZE(hsh) (RHASH(hsh)->tbl->num_entries)
#endif
#ifndef RFLOAT_VALUE
#define RFLOAT_VALUE(val) (RFLOAT(val)->value)
#endif
#ifndef RARRAY_PTR
#define RARRAY_PTR(ARRAY) RARRAY(ARRAY)->ptr
#endif
#ifndef RARRAY_LEN
#define RARRAY_LEN(ARRAY) RARRAY(ARRAY)->len
#endif
#ifndef RSTRING_PTR
#define RSTRING_PTR(string) RSTRING(string)->ptr
#endif
#ifndef RSTRING_LEN
#define RSTRING_LEN(string) RSTRING(string)->len
#endif
#define RSTRING_PAIR(string) RSTRING_PTR(string), RSTRING_LEN(string)
/* fbuffer implementation */
typedef struct FBufferStruct {
unsigned int initial_length;
char *ptr;
unsigned int len;
unsigned int capa;
} FBuffer;
#define FBUFFER_INITIAL_LENGTH 4096
#define FBUFFER_PTR(fb) (fb->ptr)
#define FBUFFER_LEN(fb) (fb->len)
#define FBUFFER_CAPA(fb) (fb->capa)
#define FBUFFER_PAIR(fb) FBUFFER_PTR(fb), FBUFFER_LEN(fb)
static char *fstrndup(const char *ptr, int len);
static FBuffer *fbuffer_alloc();
static FBuffer *fbuffer_alloc_with_length(unsigned initial_length);
static void fbuffer_free(FBuffer *fb);
static void fbuffer_free_only_buffer(FBuffer *fb);
static void fbuffer_clear(FBuffer *fb);
static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned int len);
static void fbuffer_append_long(FBuffer *fb, long number);
static void fbuffer_append_char(FBuffer *fb, char newchr);
static FBuffer *fbuffer_dup(FBuffer *fb);
/* unicode defintions */
#define UNI_STRICT_CONVERSION 1
typedef unsigned long UTF32; /* at least 32 bits */
typedef unsigned short UTF16; /* at least 16 bits */
typedef unsigned char UTF8; /* typically 8 bits */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
#define UNI_SUR_HIGH_START (UTF32)0xD800
#define UNI_SUR_HIGH_END (UTF32)0xDBFF
#define UNI_SUR_LOW_START (UTF32)0xDC00
#define UNI_SUR_LOW_END (UTF32)0xDFFF
static const int halfShift = 10; /* used for shifting by 10 bits */
static const UTF32 halfBase = 0x0010000UL;
static const UTF32 halfMask = 0x3FFUL;
static unsigned char isLegalUTF8(const UTF8 *source, int length);
static void unicode_escape(char *buf, UTF16 character);
static void unicode_escape_to_buffer(FBuffer *buffer, char buf[6], UTF16 character);
static void convert_UTF8_to_JSON_ASCII(FBuffer *buffer, VALUE string);
static void convert_UTF8_to_JSON(FBuffer *buffer, VALUE string);
/* ruby api and some helpers */
typedef struct JSON_Generator_StateStruct {
char *indent;
long indent_len;
char *space;
long space_len;
char *space_before;
long space_before_len;
char *object_nl;
long object_nl_len;
char *array_nl;
long array_nl_len;
FBuffer *array_delim;
FBuffer *object_delim;
FBuffer *object_delim2;
long max_nesting;
char allow_nan;
char ascii_only;
} JSON_Generator_State;
#define GET_STATE(self) \
JSON_Generator_State *state; \
Data_Get_Struct(self, JSON_Generator_State, state)
static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mString_included_s(VALUE self, VALUE modul);
static VALUE mString_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mString_to_json_raw_object(VALUE self);
static VALUE mString_to_json_raw(int argc, VALUE *argv, VALUE self);
static VALUE mString_Extend_json_create(VALUE self, VALUE o);
static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mNilClass_to_json(int argc, VALUE *argv, VALUE self);
static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self);
static void State_free(JSON_Generator_State *state);
static JSON_Generator_State *State_allocate();
static VALUE cState_s_allocate(VALUE klass);
static VALUE cState_configure(VALUE self, VALUE opts);
static VALUE cState_to_h(VALUE self);
static void generate_json(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj, long depth);
static VALUE cState_partial_generate(VALUE self, VALUE obj, VALUE depth);
static VALUE cState_generate(VALUE self, VALUE obj);
static VALUE cState_initialize(int argc, VALUE *argv, VALUE self);
static VALUE cState_from_state_s(VALUE self, VALUE opts);
static VALUE cState_indent(VALUE self);
static VALUE cState_indent_set(VALUE self, VALUE indent);
static VALUE cState_space(VALUE self);
static VALUE cState_space_set(VALUE self, VALUE space);
static VALUE cState_space_before(VALUE self);
static VALUE cState_space_before_set(VALUE self, VALUE space_before);
static VALUE cState_object_nl(VALUE self);
static VALUE cState_object_nl_set(VALUE self, VALUE object_nl);
static VALUE cState_array_nl(VALUE self);
static VALUE cState_array_nl_set(VALUE self, VALUE array_nl);
static VALUE cState_max_nesting(VALUE self);
static VALUE cState_max_nesting_set(VALUE self, VALUE depth);
static VALUE cState_allow_nan_p(VALUE self);
static VALUE cState_ascii_only_p(VALUE self);
#endif

27
ext/json/lib/json/add/core.rb
View file

@ -7,6 +7,19 @@ unless Object.const_defined?(:JSON) and ::JSON.const_defined?(:JSON_LOADED) and
end
require 'date'
class Symbol
def to_json(*a)
{
JSON.create_id => self.class.name,
's' => to_s,
}.to_json(*a)
end
def self.json_create(o)
o['s'].to_sym
end
end
class Time
def self.json_create(object)
if usec = object.delete('u') # used to be tv_usec -> tv_nsec
@ -21,7 +34,7 @@ class Time
def to_json(*args)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
's' => tv_sec,
'n' => respond_to?(:tv_nsec) ? tv_nsec : tv_usec * 1000
}.to_json(*args)
@ -37,7 +50,7 @@ class Date
def to_json(*args)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
'y' => year,
'm' => month,
'd' => day,
@ -63,7 +76,7 @@ class DateTime
def to_json(*args)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
'y' => year,
'm' => month,
'd' => day,
@ -83,7 +96,7 @@ class Range
def to_json(*args)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
'a' => [ first, last, exclude_end? ]
}.to_json(*args)
end
@ -98,7 +111,7 @@ class Struct
klass = self.class.name
klass.to_s.empty? and raise JSON::JSONError, "Only named structs are supported!"
{
'json_class' => klass,
JSON.create_id => klass,
'v' => values,
}.to_json(*args)
end
@ -113,7 +126,7 @@ class Exception
def to_json(*args)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
'm' => message,
'b' => backtrace,
}.to_json(*args)
@ -127,7 +140,7 @@ class Regexp
def to_json(*)
{
'json_class' => self.class.name,
JSON.create_id => self.class.name,
'o' => options,
's' => source,
}.to_json

4
ext/json/lib/json/add/rails.rb
View file

@ -10,7 +10,7 @@ class Object
def self.json_create(object)
obj = new
for key, value in object
next if key == 'json_class'
next if key == JSON.create_id
instance_variable_set "@#{key}", value
end
obj
@ -18,7 +18,7 @@ class Object
def to_json(*a)
result = {
'json_class' => self.class.name
JSON.create_id => self.class.name
}
instance_variables.inject(result) do |r, name|
r[name[1..-1]] = instance_variable_get name

147
ext/json/lib/json/common.rb
View file

@ -1,4 +1,5 @@
require 'json/version'
require 'iconv'
module JSON
class << self
@ -32,12 +33,16 @@ module JSON
# level (absolute namespace path?). If there doesn't exist a constant at
# the given path, an ArgumentError is raised.
def deep_const_get(path) # :nodoc:
path = path.to_s
path.split(/::/).inject(Object) do |p, c|
path.to_s.split(/::/).inject(Object) do |p, c|
case
when c.empty? then p
when p.const_defined?(c) then p.const_get(c)
else raise ArgumentError, "can't find const #{path}"
else
begin
p.const_missing(c)
rescue NameError
raise ArgumentError, "can't find const #{path}"
end
end
end
end
@ -58,6 +63,20 @@ module JSON
end
self.state = generator::State
const_set :State, self.state
const_set :SAFE_STATE_PROTOTYPE, State.new.freeze
const_set :FAST_STATE_PROTOTYPE, State.new(
:indent => '',
:space => '',
:object_nl => "",
:array_nl => "",
:max_nesting => false
).freeze
const_set :PRETTY_STATE_PROTOTYPE, State.new(
:indent => ' ',
:space => ' ',
:object_nl => "\n",
:array_nl => "\n"
).freeze
end
# Returns the JSON generator modul, that is used by JSON. This might be
@ -90,22 +109,22 @@ module JSON
# deep.
class NestingError < ParserError; end
# :stopdoc:
class CircularDatastructure < NestingError; end
# :startdoc:
# This exception is raised, if a generator or unparser error occurs.
class GeneratorError < JSONError; end
# For backwards compatibility
UnparserError = GeneratorError
# If a circular data structure is encountered while unparsing
# this exception is raised.
class CircularDatastructure < GeneratorError; end
# This exception is raised, if the required unicode support is missing on the
# system. Usually this means, that the iconv library is not installed.
class MissingUnicodeSupport < JSONError; end
module_function
# Parse the JSON string _source_ into a Ruby data structure and return it.
# Parse the JSON document _source_ into a Ruby data structure and return it.
#
# _opts_ can have the following
# keys:
@ -115,16 +134,21 @@ module JSON
# * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
# defiance of RFC 4627 to be parsed by the Parser. This option defaults
# to false.
# * *symbolize_names*: If set to true, returns symbols for the names
# (keys) in a JSON object. Otherwise strings are returned, which is also
# the default.
# * *create_additions*: If set to false, the Parser doesn't create
# additions even if a matchin class and create_id was found. This option
# defaults to true.
# * *object_class*: Defaults to Hash
# * *array_class*: Defaults to Array
def parse(source, opts = {})
JSON.parser.new(source, opts).parse
Parser.new(source, opts).parse
end
# Parse the JSON string _source_ into a Ruby data structure and return it.
# Parse the JSON document _source_ into a Ruby data structure and return it.
# The bang version of the parse method, defaults to the more dangerous values
# for the _opts_ hash, so be sure only to parse trusted _source_ strings.
# for the _opts_ hash, so be sure only to parse trusted _source_ documents.
#
# _opts_ can have the following keys:
# * *max_nesting*: The maximum depth of nesting allowed in the parsed data
@ -139,15 +163,14 @@ module JSON
# defaults to true.
def parse!(source, opts = {})
opts = {
:max_nesting => false,
:allow_nan => true
:max_nesting => false,
:allow_nan => true
}.update(opts)
JSON.parser.new(source, opts).parse
Parser.new(source, opts).parse
end
# Unparse the Ruby data structure _obj_ into a single line JSON string and
# return it. _state_ is
# * a JSON::State object,
# Generate a JSON document from the Ruby data structure _obj_ and return
# it. _state_ is * a JSON::State object,
# * or a Hash like object (responding to to_hash),
# * an object convertible into a hash by a to_h method,
# that is used as or to configure a State object.
@ -160,10 +183,8 @@ module JSON
# * *indent*: a string used to indent levels (default: ''),
# * *space*: a string that is put after, a : or , delimiter (default: ''),
# * *space_before*: a string that is put before a : pair delimiter (default: ''),
# * *object_nl*: a string that is put at the end of a JSON object (default: ''),
# * *object_nl*: a string that is put at the end of a JSON object (default: ''),
# * *array_nl*: a string that is put at the end of a JSON array (default: ''),
# * *check_circular*: true if checking for circular data structures
# should be done (the default), false otherwise.
# * *allow_nan*: true if NaN, Infinity, and -Infinity should be
# generated, otherwise an exception is thrown, if these values are
# encountered. This options defaults to false.
@ -174,13 +195,21 @@ module JSON
# See also the fast_generate for the fastest creation method with the least
# amount of sanity checks, and the pretty_generate method for some
# defaults for a pretty output.
def generate(obj, state = nil)
if state
state = State.from_state(state)
def generate(obj, opts = nil)
if opts
if opts.respond_to? :to_hash
opts = opts.to_hash
elsif opts.respond_to? :to_h
opts = opts.to_h
else
raise TypeError, "can't convert #{opts.class} into Hash"
end
state = SAFE_STATE_PROTOTYPE.dup
state = state.configure(opts)
else
state = State.new
state = SAFE_STATE_PROTOTYPE
end
obj.to_json(state)
state.generate(obj)
end
# :stopdoc:
@ -190,35 +219,12 @@ module JSON
module_function :unparse
# :startdoc:
# Unparse the Ruby data structure _obj_ into a single line JSON string and
# return it. This method disables the checks for circles in Ruby objects, and
# also generates NaN, Infinity, and, -Infinity float values.
# Generate a JSON document from the Ruby data structure _obj_ and return it.
# This method disables the checks for circles in Ruby objects.
#
# *WARNING*: Be careful not to pass any Ruby data structures with circles as
# _obj_ argument, because this will cause JSON to go into an infinite loop.
def fast_generate(obj)
obj.to_json(nil)
end
# :stopdoc:
# I want to deprecate these later, so I'll first be silent about them, and later delete them.
alias fast_unparse fast_generate
module_function :fast_unparse
# :startdoc:
# Unparse the Ruby data structure _obj_ into a JSON string and return it. The
# returned string is a prettier form of the string returned by #unparse.
#
# The _opts_ argument can be used to configure the generator, see the
# generate method for a more detailed explanation.
def pretty_generate(obj, opts = nil)
state = JSON.state.new(
:indent => ' ',
:space => ' ',
:object_nl => "\n",
:array_nl => "\n",
:check_circular => true
)
def fast_generate(obj, opts = nil)
if opts
if opts.respond_to? :to_hash
opts = opts.to_hash
@ -227,9 +233,41 @@ module JSON
else
raise TypeError, "can't convert #{opts.class} into Hash"
end
state = FAST_STATE_PROTOTYPE.dup
state.configure(opts)
else
state = FAST_STATE_PROTOTYPE
end
obj.to_json(state)
state.generate(obj)
end
# :stopdoc:
# I want to deprecate these later, so I'll first be silent about them, and later delete them.
alias fast_unparse fast_generate
module_function :fast_unparse
# :startdoc:
# Generate a JSON document from the Ruby data structure _obj_ and return it.
# The returned document is a prettier form of the document returned by
# #unparse.
#
# The _opts_ argument can be used to configure the generator, see the
# generate method for a more detailed explanation.
def pretty_generate(obj, opts = nil)
if opts
if opts.respond_to? :to_hash
opts = opts.to_hash
elsif opts.respond_to? :to_h
opts = opts.to_h
else
raise TypeError, "can't convert #{opts.class} into Hash"
end
state = PRETTY_STATE_PROTOTYPE.dup
state.configure(opts)
else
state = PRETTY_STATE_PROTOTYPE
end
state.generate(obj)
end
# :stopdoc:
@ -305,6 +343,11 @@ module JSON
rescue JSON::NestingError
raise ArgumentError, "exceed depth limit"
end
# Shortuct for iconv.
def self.iconv(to, from, string)
Iconv.iconv(to, from, string).first
end
end
module ::Kernel

32
ext/json/lib/json/editor.rb
View file

@ -48,14 +48,14 @@ module JSON
# Opens an error dialog on top of _window_ showing the error message
# _text_.
def Editor.error_dialog(window, text)
dialog = MessageDialog.new(window, Dialog::MODAL,
MessageDialog::ERROR,
dialog = MessageDialog.new(window, Dialog::MODAL,
MessageDialog::ERROR,
MessageDialog::BUTTONS_CLOSE, text)
dialog.show_all
dialog.run
rescue TypeError
dialog = MessageDialog.new(Editor.window, Dialog::MODAL,
MessageDialog::ERROR,
dialog = MessageDialog.new(Editor.window, Dialog::MODAL,
MessageDialog::ERROR,
MessageDialog::BUTTONS_CLOSE, text)
dialog.show_all
dialog.run
@ -67,8 +67,8 @@ module JSON
# message _text_. If yes was answered _true_ is returned, otherwise
# _false_.
def Editor.question_dialog(window, text)
dialog = MessageDialog.new(window, Dialog::MODAL,
MessageDialog::QUESTION,
dialog = MessageDialog.new(window, Dialog::MODAL,
MessageDialog::QUESTION,
MessageDialog::BUTTONS_YES_NO, text)
dialog.show_all
dialog.run do |response|
@ -465,7 +465,7 @@ module JSON
add_separator
add_item("Append new node", ?a, &method(:append_new_node))
add_item("Insert new node before", ?i, &method(:insert_new_node))
add_separator
add_separator
add_item("Collapse/Expand node (recursively)", ?e,
&method(:collapse_expand))
@ -504,7 +504,7 @@ module JSON
# Revert the current JSON document in the editor to the saved version.
def revert(item)
window.instance_eval do
@filename and file_open(@filename)
@filename and file_open(@filename)
end
end
@ -666,7 +666,7 @@ module JSON
collapse_all
else
self.expanded = true
expand_all
expand_all
end
end
@ -885,7 +885,7 @@ module JSON
dialog.signal_connect(:'key-press-event', &DEFAULT_DIALOG_KEY_PRESS_HANDLER)
dialog.show_all
self.focus = dialog
dialog.run do |response|
dialog.run do |response|
if response == Dialog::RESPONSE_ACCEPT
@key = key_input.text
type = ALL_TYPES[@type = type_input.active]
@ -937,7 +937,7 @@ module JSON
dialog.signal_connect(:'key-press-event', &DEFAULT_DIALOG_KEY_PRESS_HANDLER)
dialog.show_all
self.focus = dialog
dialog.run do |response|
dialog.run do |response|
if response == Dialog::RESPONSE_ACCEPT
type = types[type_input.active]
@content = case type
@ -982,7 +982,7 @@ module JSON
dialog.signal_connect(:'key-press-event', &DEFAULT_DIALOG_KEY_PRESS_HANDLER)
dialog.show_all
self.focus = dialog
dialog.run do |response|
dialog.run do |response|
if response == Dialog::RESPONSE_ACCEPT
return @order = order_input.text, reverse_checkbox.active?
end
@ -1017,7 +1017,7 @@ module JSON
dialog.signal_connect(:'key-press-event', &DEFAULT_DIALOG_KEY_PRESS_HANDLER)
dialog.show_all
self.focus = dialog
dialog.run do |response|
dialog.run do |response|
if response == Dialog::RESPONSE_ACCEPT
begin
return Regexp.new(regex_input.text, icase_checkbox.active? ? Regexp::IGNORECASE : 0)
@ -1216,7 +1216,7 @@ module JSON
end
end
# Save the current file as the filename
# Save the current file as the filename
def file_save_as
filename = select_file('Save as a JSON file')
store_file(filename)
@ -1242,7 +1242,7 @@ module JSON
rescue SystemCallError => e
Editor.error_dialog(self, "Failed to store JSON file: #{e}!")
end
# Load the file named _filename_ into the editor as a JSON document.
def load_file(filename)
if filename
@ -1335,7 +1335,7 @@ module JSON
dialog.signal_connect(:'key-press-event', &DEFAULT_DIALOG_KEY_PRESS_HANDLER)
dialog.show_all
dialog.run do |response|
dialog.run do |response|
if response == Dialog::RESPONSE_ACCEPT
return @location = location_input.text
end

2
ext/json/lib/json/version.rb
View file

@ -1,6 +1,6 @@
module JSON
# JSON version
VERSION = '1.1.8'
VERSION = '1.4.1'
VERSION_ARRAY = VERSION.split(/\./).map { |x| x.to_i } # :nodoc:
VERSION_MAJOR = VERSION_ARRAY[0] # :nodoc:
VERSION_MINOR = VERSION_ARRAY[1] # :nodoc:

4
ext/json/parser/extconf.rb Normal file
View file

@ -0,0 +1,4 @@
require 'mkmf'
require 'rbconfig'
create_makefile 'json/ext/parser'

482
ext/json/ext/parser/parser.c → ext/json/parser/parser.c
View file

File diff suppressed because it is too large Load diff

71
ext/json/parser/parser.h Normal file
View file

@ -0,0 +1,71 @@
#ifndef _PARSER_H_
#define _PARSER_H_
#include "ruby.h"
#if HAVE_RE_H
#include "re.h"
#endif
#ifdef HAVE_RUBY_ENCODING_H
#include "ruby/encoding.h"
#define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
#else
#define FORCE_UTF8(obj)
#endif
#define option_given_p(opts, key) RTEST(rb_funcall(opts, i_key_p, 1, key))
/* unicode */
typedef unsigned long UTF32; /* at least 32 bits */
typedef unsigned short UTF16; /* at least 16 bits */
typedef unsigned char UTF8; /* typically 8 bits */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_SUR_HIGH_START (UTF32)0xD800
#define UNI_SUR_HIGH_END (UTF32)0xDBFF
#define UNI_SUR_LOW_START (UTF32)0xDC00
#define UNI_SUR_LOW_END (UTF32)0xDFFF
typedef struct JSON_ParserStruct {
VALUE Vsource;
char *source;
long len;
char *memo;
VALUE create_id;
int max_nesting;
int current_nesting;
int allow_nan;
int parsing_name;
int symbolize_names;
VALUE object_class;
VALUE array_class;
} JSON_Parser;
#define GET_PARSER \
JSON_Parser *json; \
Data_Get_Struct(self, JSON_Parser, json)
#define MinusInfinity "-Infinity"
#define EVIL 0x666
static UTF32 unescape_unicode(const unsigned char *p);
static int convert_UTF32_to_UTF8(char *buf, UTF32 ch);
static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_value(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_integer(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *result);
static VALUE json_string_unescape(VALUE result, char *string, char *stringEnd);
static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *result);
static VALUE convert_encoding(VALUE source);
static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self);
static VALUE cParser_parse(VALUE self);
static JSON_Parser *JSON_allocate();
static void JSON_mark(JSON_Parser *json);
static void JSON_free(JSON_Parser *json);
static VALUE cJSON_parser_s_allocate(VALUE klass);
static VALUE cParser_source(VALUE self);
#endif

334
ext/json/ext/parser/parser.rl → ext/json/parser/parser.rl
View file

@ -1,59 +1,83 @@
#include "ruby.h"
#include "unicode.h"
#if HAVE_RE_H
#include "re.h"
#endif
#if HAVE_RUBY_ST_H
#include "ruby/st.h"
#endif
#if HAVE_ST_H
#include "st.h"
#endif
#include "parser.h"
#define EVIL 0x666
/* unicode */
#ifndef RHASH_TBL
#define RHASH_TBL(hsh) (RHASH(hsh)->tbl)
#endif
static const char digit_values[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1,
-1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1
};
static UTF32 unescape_unicode(const unsigned char *p)
{
char b;
UTF32 result = 0;
b = digit_values[p[0]];
if (b < 0) return UNI_REPLACEMENT_CHAR;
result = (result << 4) | b;
b = digit_values[p[1]];
result = (result << 4) | b;
if (b < 0) return UNI_REPLACEMENT_CHAR;
b = digit_values[p[2]];
result = (result << 4) | b;
if (b < 0) return UNI_REPLACEMENT_CHAR;
b = digit_values[p[3]];
result = (result << 4) | b;
if (b < 0) return UNI_REPLACEMENT_CHAR;
return result;
}
static int convert_UTF32_to_UTF8(char *buf, UTF32 ch)
{
int len = 1;
if (ch <= 0x7F) {
buf[0] = (char) ch;
} else if (ch <= 0x07FF) {
buf[0] = (char) ((ch >> 6) | 0xC0);
buf[1] = (char) ((ch & 0x3F) | 0x80);
len++;
} else if (ch <= 0xFFFF) {
buf[0] = (char) ((ch >> 12) | 0xE0);
buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
buf[2] = (char) ((ch & 0x3F) | 0x80);
len += 2;
} else if (ch <= 0x1fffff) {
buf[0] =(char) ((ch >> 18) | 0xF0);
buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
buf[3] =(char) ((ch & 0x3F) | 0x80);
len += 3;
} else {
buf[0] = '?';
}
return len;
}
#ifdef HAVE_RUBY_ENCODING_H
#include "ruby/encoding.h"
#define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
static VALUE CEncoding_ASCII_8BIT, CEncoding_UTF_8, CEncoding_UTF_16BE,
CEncoding_UTF_16LE, CEncoding_UTF_32BE, CEncoding_UTF_32LE;
static ID i_encoding, i_encode, i_encode_bang, i_force_encoding;
#else
#define FORCE_UTF8(obj)
static ID i_iconv;
#endif
static VALUE mJSON, mExt, cParser, eParserError, eNestingError;
static VALUE CNaN, CInfinity, CMinusInfinity;
static ID i_json_creatable_p, i_json_create, i_create_id, i_create_additions,
i_chr, i_max_nesting, i_allow_nan, i_object_class, i_array_class;
#define MinusInfinity "-Infinity"
typedef struct JSON_ParserStruct {
VALUE Vsource;
char *source;
long len;
char *memo;
VALUE create_id;
int max_nesting;
int current_nesting;
int allow_nan;
VALUE object_class;
VALUE array_class;
} JSON_Parser;
static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_value(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_integer(JSON_Parser *json, char *p, char *pe, VALUE *result);
static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *result);
#define GET_STRUCT \
JSON_Parser *json; \
Data_Get_Struct(self, JSON_Parser, json);
i_chr, i_max_nesting, i_allow_nan, i_symbolize_names, i_object_class,
i_array_class, i_key_p, i_deep_const_get;
%%{
machine JSON_common;
@ -91,7 +115,7 @@ static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *resul
action parse_value {
VALUE v = Qnil;
char *np = JSON_parse_value(json, fpc, pe, &v);
char *np = JSON_parse_value(json, fpc, pe, &v);
if (np == NULL) {
fhold; fbreak;
} else {
@ -101,7 +125,10 @@ static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *resul
}
action parse_name {
char *np = JSON_parse_string(json, fpc, pe, &last_name);
char *np;
json->parsing_name = 1;
np = JSON_parse_string(json, fpc, pe, &last_name);
json->parsing_name = 0;
if (np == NULL) { fhold; fbreak; } else fexec np;
}
@ -135,7 +162,7 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
if (RTEST(json->create_id)) {
VALUE klassname = rb_hash_aref(*result, json->create_id);
if (!NIL_P(klassname)) {
VALUE klass = rb_path2class(StringValueCStr(klassname));
VALUE klass = rb_funcall(mJSON, i_deep_const_get, 1, klassname);
if RTEST(rb_funcall(klass, i_json_creatable_p, 0)) {
*result = rb_funcall(klass, i_json_create, 1, *result);
}
@ -199,7 +226,7 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
fhold; fbreak;
}
action parse_array {
action parse_array {
char *np;
json->current_nesting++;
np = JSON_parse_array(json, fpc, pe, result);
@ -207,7 +234,7 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
if (np == NULL) { fhold; fbreak; } else fexec np;
}
action parse_object {
action parse_object {
char *np;
json->current_nesting++;
np = JSON_parse_object(json, fpc, pe, result);
@ -311,7 +338,7 @@ static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *resul
action parse_value {
VALUE v = Qnil;
char *np = JSON_parse_value(json, fpc, pe, &v);
char *np = JSON_parse_value(json, fpc, pe, &v);
if (np == NULL) {
fhold; fbreak;
} else {
@ -347,62 +374,77 @@ static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *resul
return p + 1;
} else {
rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
return NULL;
}
}
static VALUE json_string_unescape(char *p, char *pe)
static VALUE json_string_unescape(VALUE result, char *string, char *stringEnd)
{
VALUE result = rb_str_buf_new(pe - p + 1);
char *p = string, *pe = string, *unescape;
int unescape_len;
while (p < pe) {
if (*p == '\\') {
p++;
if (p >= pe) return Qnil; /* raise an exception later, \ at end */
switch (*p) {
case '"':
case '\\':
rb_str_buf_cat(result, p, 1);
p++;
break;
case 'b':
rb_str_buf_cat2(result, "\b");
p++;
break;
case 'f':
rb_str_buf_cat2(result, "\f");
p++;
break;
while (pe < stringEnd) {
if (*pe == '\\') {
unescape = (char *) "?";
unescape_len = 1;
if (pe > p) rb_str_buf_cat(result, p, pe - p);
switch (*++pe) {
case 'n':
rb_str_buf_cat2(result, "\n");
p++;
unescape = (char *) "\n";
break;
case 'r':
rb_str_buf_cat2(result, "\r");
p++;
unescape = (char *) "\r";
break;
case 't':
rb_str_buf_cat2(result, "\t");
p++;
unescape = (char *) "\t";
break;
case '"':
unescape = (char *) "\"";
break;
case '\\':
unescape = (char *) "\\";
break;
case 'b':
unescape = (char *) "\b";
break;
case 'f':
unescape = (char *) "\f";
break;
case 'u':
if (p > pe - 4) {
if (pe > stringEnd - 4) {
return Qnil;
} else {
p = JSON_convert_UTF16_to_UTF8(result, p, pe, strictConversion);
char buf[4];
UTF32 ch = unescape_unicode((unsigned char *) ++pe);
pe += 3;
if (UNI_SUR_HIGH_START == (ch & 0xFC00)) {
pe++;
if (pe > stringEnd - 6) return Qnil;
if (pe[0] == '\\' && pe[1] == 'u') {
UTF32 sur = unescape_unicode((unsigned char *) pe + 2);
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16)
| (sur & 0x3FF));
pe += 5;
} else {
unescape = (char *) "?";
break;
}
}
unescape_len = convert_UTF32_to_UTF8(buf, ch);
unescape = buf;
}
break;
default:
rb_str_buf_cat(result, p, 1);
p++;
break;
p = pe;
continue;
}
rb_str_buf_cat(result, unescape, unescape_len);
p = ++pe;
} else {
char *q = p;
while (*q != '\\' && q < pe) q++;
rb_str_buf_cat(result, p, q - p);
p = q;
pe++;
}
}
rb_str_buf_cat(result, p, pe - p);
return result;
}
@ -413,7 +455,7 @@ static VALUE json_string_unescape(char *p, char *pe)
write data;
action parse_string {
*result = json_string_unescape(json->memo + 1, p);
*result = json_string_unescape(*result, json->memo + 1, p);
if (NIL_P(*result)) {
fhold;
fbreak;
@ -432,11 +474,14 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
{
int cs = EVIL;
*result = rb_str_new("", 0);
*result = rb_str_buf_new(0);
%% write init;
json->memo = p;
%% write exec;
if (json->symbolize_names && json->parsing_name) {
*result = rb_str_intern(*result);
}
if (cs >= JSON_string_first_final) {
return p + 1;
} else {
@ -472,7 +517,7 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
) ignore*;
}%%
/*
/*
* Document-class: JSON::Ext::Parser
*
* This is the JSON parser implemented as a C extension. It can be configured
@ -484,6 +529,54 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
*
*/
static VALUE convert_encoding(VALUE source)
{
char *ptr = RSTRING_PTR(source);
long len = RSTRING_LEN(source);
if (len < 2) {
rb_raise(eParserError, "A JSON text must at least contain two octets!");
}
#ifdef HAVE_RUBY_ENCODING_H
{
VALUE encoding = rb_funcall(source, i_encoding, 0);
if (encoding == CEncoding_ASCII_8BIT) {
if (len >= 4 && ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
source = rb_str_dup(source);
rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_32BE);
source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
} else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
source = rb_str_dup(source);
rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_16BE);
source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
} else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
source = rb_str_dup(source);
rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_32LE);
source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
} else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
source = rb_str_dup(source);
rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_16LE);
source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
} else {
FORCE_UTF8(source);
}
} else {
source = rb_funcall(source, i_encode, 1, CEncoding_UTF_8);
}
}
#else
if (len >= 4 && ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-32be"), source);
} else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-16be"), source);
} else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-32le"), source);
} else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
source = rb_funcall(mJSON, i_iconv, 3, rb_str_new2("utf-8"), rb_str_new2("utf-16le"), source);
}
#endif
return source;
}
/*
* call-seq: new(source, opts => {})
*
@ -501,6 +594,9 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
* * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
* defiance of RFC 4627 to be parsed by the Parser. This option defaults to
* false.
* * *symbolize_names*: If set to true, returns symbols for the names
* (keys) in a JSON object. Otherwise strings are returned, which is also
* the default.
* * *create_additions*: If set to false, the Parser doesn't create
* additions even if a matchin class and create_id was found. This option
* defaults to true.
@ -512,21 +608,18 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
char *ptr;
long len;
VALUE source, opts;
GET_STRUCT;
GET_PARSER;
rb_scan_args(argc, argv, "11", &source, &opts);
source = StringValue(source);
source = convert_encoding(StringValue(source));
ptr = RSTRING_PTR(source);
len = RSTRING_LEN(source);
if (len < 2) {
rb_raise(eParserError, "A JSON text must at least contain two octets!");
}
if (!NIL_P(opts)) {
opts = rb_convert_type(opts, T_HASH, "Hash", "to_hash");
if (NIL_P(opts)) {
rb_raise(rb_eArgError, "opts needs to be like a hash");
} else {
VALUE tmp = ID2SYM(i_max_nesting);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
if (option_given_p(opts, tmp)) {
VALUE max_nesting = rb_hash_aref(opts, tmp);
if (RTEST(max_nesting)) {
Check_Type(max_nesting, T_FIXNUM);
@ -538,14 +631,21 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
json->max_nesting = 19;
}
tmp = ID2SYM(i_allow_nan);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
if (option_given_p(opts, tmp)) {
VALUE allow_nan = rb_hash_aref(opts, tmp);
json->allow_nan = RTEST(allow_nan) ? 1 : 0;
} else {
json->allow_nan = 0;
}
tmp = ID2SYM(i_symbolize_names);
if (option_given_p(opts, tmp)) {
VALUE symbolize_names = rb_hash_aref(opts, tmp);
json->symbolize_names = RTEST(symbolize_names) ? 1 : 0;
} else {
json->symbolize_names = 0;
}
tmp = ID2SYM(i_create_additions);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
if (option_given_p(opts, tmp)) {
VALUE create_additions = rb_hash_aref(opts, tmp);
if (RTEST(create_additions)) {
json->create_id = rb_funcall(mJSON, i_create_id, 0);
@ -556,13 +656,13 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
json->create_id = rb_funcall(mJSON, i_create_id, 0);
}
tmp = ID2SYM(i_object_class);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
if (option_given_p(opts, tmp)) {
json->object_class = rb_hash_aref(opts, tmp);
} else {
json->object_class = Qnil;
}
tmp = ID2SYM(i_array_class);
if (st_lookup(RHASH_TBL(opts), tmp, 0)) {
if (option_given_p(opts, tmp)) {
json->array_class = rb_hash_aref(opts, tmp);
} else {
json->array_class = Qnil;
@ -576,18 +676,6 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
json->array_class = Qnil;
}
json->current_nesting = 0;
/*
Convert these?
if (len >= 4 && ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
} else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
} else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
} else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
rb_raise(eParserError, "Only UTF8 octet streams are supported atm!");
}
*/
json->len = len;
json->source = ptr;
json->Vsource = source;
@ -605,7 +693,7 @@ static VALUE cParser_parse(VALUE self)
char *p, *pe;
int cs = EVIL;
VALUE result = Qnil;
GET_STRUCT;
GET_PARSER;
%% write init;
p = json->source;
@ -616,10 +704,11 @@ static VALUE cParser_parse(VALUE self)
return result;
} else {
rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
return Qnil;
}
}
inline static JSON_Parser *JSON_allocate()
static JSON_Parser *JSON_allocate()
{
JSON_Parser *json = ALLOC(JSON_Parser);
MEMZERO(json, JSON_Parser, 1);
@ -653,7 +742,7 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
*/
static VALUE cParser_source(VALUE self)
{
GET_STRUCT;
GET_PARSER;
return rb_str_dup(json->Vsource);
}
@ -681,6 +770,23 @@ void Init_parser()
i_chr = rb_intern("chr");
i_max_nesting = rb_intern("max_nesting");
i_allow_nan = rb_intern("allow_nan");
i_symbolize_names = rb_intern("symbolize_names");
i_object_class = rb_intern("object_class");
i_array_class = rb_intern("array_class");
i_key_p = rb_intern("key?");
i_deep_const_get = rb_intern("deep_const_get");
#ifdef HAVE_RUBY_ENCODING_H
CEncoding_UTF_8 = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-8"));
CEncoding_UTF_16BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16be"));
CEncoding_UTF_16LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16le"));
CEncoding_UTF_32BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32be"));
CEncoding_UTF_32LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32le"));
CEncoding_ASCII_8BIT = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("ascii-8bit"));
i_encoding = rb_intern("encoding");
i_encode = rb_intern("encode");
i_encode_bang = rb_intern("encode!");
i_force_encoding = rb_intern("force_encoding");
#else
i_iconv = rb_intern("iconv");
#endif
}