mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
* include/ruby/oniguruma.h: remove ONIG_ENCODING_* and OnigEncoding*
which are not builtin. * regenc.{c,h} (onigenc_mb2_code_to_mbclen, onigenc_mb4_code_to_mbclen): fix prototype. * enc/big5.c, enc/euc_kr.c, enc/euc_tw.c, enc/gb18030.c, enc/koi8_r.c, enc/windows_1251.c: imported from Oniguruma. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@15026 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
parent
21671b558c
commit
8f15b8128c
14 changed files with 1383 additions and 74 deletions
11
ChangeLog
11
ChangeLog
|
@ -1,3 +1,14 @@
|
|||
Mon Jan 14 02:03:05 2008 NARUSE, Yui <naruse@ruby-lang.org>
|
||||
|
||||
* include/ruby/oniguruma.h: remove ONIG_ENCODING_* and OnigEncoding*
|
||||
which are not builtin.
|
||||
|
||||
* regenc.{c,h} (onigenc_mb2_code_to_mbclen, onigenc_mb4_code_to_mbclen):
|
||||
fix prototype.
|
||||
|
||||
* enc/big5.c, enc/euc_kr.c, enc/euc_tw.c, enc/gb18030.c,
|
||||
enc/koi8_r.c, enc/windows_1251.c: imported from Oniguruma.
|
||||
|
||||
Sun Jan 13 22:47:28 2008 NARUSE, Yui <naruse@ruby-lang.org>
|
||||
|
||||
* enc/make_encdb.h: sort encoding names by original name.
|
||||
|
|
164
enc/big5.c
Normal file
164
enc/big5.c
Normal file
|
@ -0,0 +1,164 @@
|
|||
/**********************************************************************
|
||||
big5.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
OnigEncodingDeclare(BIG5);
|
||||
|
||||
static const int EncLen_BIG5[] = {
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1
|
||||
};
|
||||
|
||||
static int
|
||||
big5_mbc_enc_len(const UChar* p, const UChar* e, OnigEncoding enc)
|
||||
{
|
||||
return EncLen_BIG5[*p];
|
||||
}
|
||||
|
||||
static OnigCodePoint
|
||||
big5_mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_to_code(&OnigEncodingName(BIG5), p, end);
|
||||
}
|
||||
|
||||
static int
|
||||
big5_code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb2_code_to_mbc(&OnigEncodingName(BIG5), code, buf);
|
||||
}
|
||||
|
||||
static int
|
||||
big5_mbc_case_fold(OnigCaseFoldType flag, const UChar** pp, const UChar* end,
|
||||
UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_case_fold(&OnigEncodingName(BIG5), flag,
|
||||
pp, end, lower);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
big5_is_mbc_ambiguous(OnigCaseFoldType flag,
|
||||
const UChar** pp, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_is_mbc_ambiguous(&OnigEncodingName(BIG5), flag, pp, end);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
big5_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb2_is_code_ctype(&OnigEncodingName(BIG5), code, ctype);
|
||||
}
|
||||
|
||||
static const char BIG5_CAN_BE_TRAIL_TABLE[256] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0
|
||||
};
|
||||
|
||||
#define BIG5_ISMB_FIRST(byte) (EncLen_BIG5[byte] > 1)
|
||||
#define BIG5_ISMB_TRAIL(byte) BIG5_CAN_BE_TRAIL_TABLE[(byte)]
|
||||
|
||||
static UChar*
|
||||
big5_left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
||||
{
|
||||
const UChar *p;
|
||||
int len;
|
||||
|
||||
if (s <= start) return (UChar* )s;
|
||||
p = s;
|
||||
|
||||
if (BIG5_ISMB_TRAIL(*p)) {
|
||||
while (p > start) {
|
||||
if (! BIG5_ISMB_FIRST(*--p)) {
|
||||
p++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
len = enclen(&OnigEncodingName(BIG5), p, s);
|
||||
if (p + len > s) return (UChar* )p;
|
||||
p += len;
|
||||
return (UChar* )(p + ((s - p) & ~1));
|
||||
}
|
||||
|
||||
static int
|
||||
big5_is_allowed_reverse_match(const UChar* s, const UChar* end ARG_UNUSED, OnigEncoding enc)
|
||||
{
|
||||
const UChar c = *s;
|
||||
|
||||
return (BIG5_ISMB_TRAIL(c) ? FALSE : TRUE);
|
||||
}
|
||||
|
||||
OnigEncodingDefine(big5, BIG5) = {
|
||||
big5_mbc_enc_len,
|
||||
"Big5", /* name */
|
||||
2, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
big5_mbc_to_code,
|
||||
onigenc_mb2_code_to_mbclen,
|
||||
big5_code_to_mbc,
|
||||
big5_mbc_case_fold,
|
||||
onigenc_ascii_apply_all_case_fold,
|
||||
onigenc_ascii_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
big5_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
big5_left_adjust_char_head,
|
||||
big5_is_allowed_reverse_match
|
||||
};
|
|
@ -139,7 +139,7 @@ mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
|||
int c, i, len;
|
||||
OnigCodePoint n;
|
||||
|
||||
len = enclen(&encoding_EUC_JP, p, end);
|
||||
len = enclen(&OnigEncodingName(EUC_JP), p, end);
|
||||
n = (OnigCodePoint )*p++;
|
||||
if (len == 1) return n;
|
||||
|
||||
|
@ -191,7 +191,7 @@ code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
|||
*p++ = (UChar )(code & 0xff);
|
||||
|
||||
#if 1
|
||||
if (enclen(&encoding_EUC_JP, buf, p) != (p - buf))
|
||||
if (enclen(&OnigEncodingName(EUC_JP), buf, p) != (p - buf))
|
||||
return ONIGERR_INVALID_CODE_POINT_VALUE;
|
||||
#endif
|
||||
return p - buf;
|
||||
|
@ -213,7 +213,7 @@ mbc_case_fold(OnigCaseFoldType flag,
|
|||
else {
|
||||
int i;
|
||||
|
||||
len = enclen(&encoding_EUC_JP, p, end);
|
||||
len = enclen(&OnigEncodingName(EUC_JP), p, end);
|
||||
for (i = 0; i < len; i++) {
|
||||
*lower++ = *p++;
|
||||
}
|
||||
|
@ -235,7 +235,7 @@ left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
|||
p = s;
|
||||
|
||||
while (!eucjp_islead(*p) && p > start) p--;
|
||||
len = enclen(&encoding_EUC_JP, p, s);
|
||||
len = enclen(&OnigEncodingName(EUC_JP), p, s);
|
||||
if (p + len > s) return (UChar* )p;
|
||||
p += len;
|
||||
return (UChar* )(p + ((s - p) & ~1));
|
||||
|
|
141
enc/euc_kr.c
Normal file
141
enc/euc_kr.c
Normal file
|
@ -0,0 +1,141 @@
|
|||
/**********************************************************************
|
||||
euc_kr.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
OnigEncodingDeclare(EUC_KR);
|
||||
|
||||
static const int EncLen_EUCKR[] = {
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1
|
||||
};
|
||||
|
||||
static int
|
||||
euckr_mbc_enc_len(const UChar* p, const UChar* e, OnigEncoding enc)
|
||||
{
|
||||
return EncLen_EUCKR[*p];
|
||||
}
|
||||
|
||||
static OnigCodePoint
|
||||
euckr_mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_to_code(&OnigEncodingName(EUC_KR), p, end);
|
||||
}
|
||||
|
||||
static int
|
||||
euckr_code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb2_code_to_mbc(&OnigEncodingName(EUC_KR), code, buf);
|
||||
}
|
||||
|
||||
static int
|
||||
euckr_mbc_case_fold(OnigCaseFoldType flag, const UChar** pp, const UChar* end,
|
||||
UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_case_fold(&OnigEncodingName(EUC_KR), flag,
|
||||
pp, end, lower);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
euckr_is_mbc_ambiguous(OnigCaseFoldType flag,
|
||||
const UChar** pp, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_is_mbc_ambiguous(&OnigEncodingName(EUC_KR), flag, pp, end);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
euckr_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb2_is_code_ctype(&OnigEncodingName(EUC_KR), code, ctype);
|
||||
}
|
||||
|
||||
#define euckr_islead(c) ((c) < 0xa1 || (c) == 0xff)
|
||||
|
||||
static UChar*
|
||||
euckr_left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
||||
{
|
||||
/* Assumed in this encoding,
|
||||
mb-trail bytes don't mix with single bytes.
|
||||
*/
|
||||
const UChar *p;
|
||||
int len;
|
||||
|
||||
if (s <= start) return (UChar* )s;
|
||||
p = s;
|
||||
|
||||
while (!euckr_islead(*p) && p > start) p--;
|
||||
len = enclen(&OnigEncodingName(EUC_KR), p, s);
|
||||
if (p + len > s) return (UChar* )p;
|
||||
p += len;
|
||||
return (UChar* )(p + ((s - p) & ~1));
|
||||
}
|
||||
|
||||
static int
|
||||
euckr_is_allowed_reverse_match(const UChar* s, const UChar* end ARG_UNUSED, OnigEncoding enc)
|
||||
{
|
||||
const UChar c = *s;
|
||||
if (c <= 0x7e) return TRUE;
|
||||
else return FALSE;
|
||||
}
|
||||
|
||||
OnigEncodingDefine(euc_kr, EUC_KR) = {
|
||||
euckr_mbc_enc_len,
|
||||
"EUC-KR", /* name */
|
||||
2, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
euckr_mbc_to_code,
|
||||
onigenc_mb2_code_to_mbclen,
|
||||
euckr_code_to_mbc,
|
||||
euckr_mbc_case_fold,
|
||||
onigenc_ascii_apply_all_case_fold,
|
||||
onigenc_ascii_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
euckr_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
euckr_left_adjust_char_head,
|
||||
euckr_is_allowed_reverse_match
|
||||
};
|
||||
ENC_REPLICATE("EUC-CN", "EUC-KR");
|
140
enc/euc_tw.c
Normal file
140
enc/euc_tw.c
Normal file
|
@ -0,0 +1,140 @@
|
|||
/**********************************************************************
|
||||
euc_tw.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
OnigEncodingDeclare(EUC_TW);
|
||||
|
||||
static const int EncLen_EUCTW[] = {
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1
|
||||
};
|
||||
|
||||
static int
|
||||
euctw_mbc_enc_len(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return EncLen_EUCTW[*p];
|
||||
}
|
||||
|
||||
static OnigCodePoint
|
||||
euctw_mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_to_code(&OnigEncodingName(EUC_TW), p, end);
|
||||
}
|
||||
|
||||
static int
|
||||
euctw_code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb4_code_to_mbc(&OnigEncodingName(EUC_TW), code, buf);
|
||||
}
|
||||
|
||||
static int
|
||||
euctw_mbc_case_fold(OnigCaseFoldType flag, const UChar** pp, const UChar* end,
|
||||
UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_case_fold(&OnigEncodingName(EUC_TW), flag,
|
||||
pp, end, lower);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
euctw_is_mbc_ambiguous(OnigCaseFoldType flag,
|
||||
const UChar** pp, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_is_mbc_ambiguous(&OnigEncodingName(EUC_TW), flag, pp, end);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
euctw_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb4_is_code_ctype(&OnigEncodingName(EUC_TW), code, ctype);
|
||||
}
|
||||
|
||||
#define euctw_islead(c) (((c) < 0xa1 && (c) != 0x8e) || (c) == 0xff)
|
||||
|
||||
static UChar*
|
||||
euctw_left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
||||
{
|
||||
/* Assumed in this encoding,
|
||||
mb-trail bytes don't mix with single bytes.
|
||||
*/
|
||||
const UChar *p;
|
||||
int len;
|
||||
|
||||
if (s <= start) return (UChar* )s;
|
||||
p = s;
|
||||
|
||||
while (!euctw_islead(*p) && p > start) p--;
|
||||
len = enclen(&OnigEncodingName(EUC_TW), p, s);
|
||||
if (p + len > s) return (UChar* )p;
|
||||
p += len;
|
||||
return (UChar* )(p + ((s - p) & ~1));
|
||||
}
|
||||
|
||||
static int
|
||||
euctw_is_allowed_reverse_match(const UChar* s, const UChar* end ARG_UNUSED, OnigEncoding enc)
|
||||
{
|
||||
const UChar c = *s;
|
||||
if (c <= 0x7e) return TRUE;
|
||||
else return FALSE;
|
||||
}
|
||||
|
||||
OnigEncodingDefine(euc_tw, EUC_TW) = {
|
||||
euctw_mbc_enc_len,
|
||||
"EUC-TW", /* name */
|
||||
4, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
euctw_mbc_to_code,
|
||||
onigenc_mb4_code_to_mbclen,
|
||||
euctw_code_to_mbc,
|
||||
euctw_mbc_case_fold,
|
||||
onigenc_ascii_apply_all_case_fold,
|
||||
onigenc_ascii_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
euctw_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
euctw_left_adjust_char_head,
|
||||
euctw_is_allowed_reverse_match
|
||||
};
|
500
enc/gb18030.c
Normal file
500
enc/gb18030.c
Normal file
|
@ -0,0 +1,500 @@
|
|||
/**********************************************************************
|
||||
gb18030.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2005-2007 KUBO Takehiro <kubo AT jiubao DOT org>
|
||||
* K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
OnigEncodingDeclare(GB18030);
|
||||
|
||||
#if 1
|
||||
#define DEBUG_GB18030(arg)
|
||||
#else
|
||||
#define DEBUG_GB18030(arg) printf arg
|
||||
#endif
|
||||
|
||||
enum {
|
||||
C1, /* one-byte char */
|
||||
C2, /* one-byte or second of two-byte char */
|
||||
C4, /* one-byte or second or fourth of four-byte char */
|
||||
CM /* first of two- or four-byte char or second of two-byte char */
|
||||
};
|
||||
|
||||
static const char GB18030_MAP[] = {
|
||||
C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1,
|
||||
C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1,
|
||||
C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1, C1,
|
||||
C4, C4, C4, C4, C4, C4, C4, C4, C4, C4, C1, C1, C1, C1, C1, C1,
|
||||
C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2,
|
||||
C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2,
|
||||
C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2,
|
||||
C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C2, C1,
|
||||
C2, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM,
|
||||
CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, CM, C1
|
||||
};
|
||||
|
||||
static int
|
||||
gb18030_mbc_enc_len(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
if (GB18030_MAP[*p] != CM)
|
||||
return 1;
|
||||
p++;
|
||||
if (GB18030_MAP[*p] == C4)
|
||||
return 4;
|
||||
if (GB18030_MAP[*p] == C1)
|
||||
return 1; /* illegal sequence */
|
||||
return 2;
|
||||
}
|
||||
|
||||
static OnigCodePoint
|
||||
gb18030_mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_to_code(&OnigEncodingName(GB18030), p, end);
|
||||
}
|
||||
|
||||
static int
|
||||
gb18030_code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb4_code_to_mbc(&OnigEncodingName(GB18030), code, buf);
|
||||
}
|
||||
|
||||
static int
|
||||
gb18030_mbc_case_fold(OnigCaseFoldType flag, const UChar** pp, const UChar* end,
|
||||
UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_mbc_case_fold(&OnigEncodingName(GB18030), flag,
|
||||
pp, end, lower);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
gb18030_is_mbc_ambiguous(OnigCaseFoldType flag,
|
||||
const UChar** pp, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mbn_is_mbc_ambiguous(&OnigEncodingName(GB18030), flag, pp, end);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
gb18030_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_mb4_is_code_ctype(&OnigEncodingName(GB18030), code, ctype);
|
||||
}
|
||||
|
||||
enum state {
|
||||
S_START,
|
||||
S_one_C2,
|
||||
S_one_C4,
|
||||
S_one_CM,
|
||||
|
||||
S_odd_CM_one_CX,
|
||||
S_even_CM_one_CX,
|
||||
|
||||
/* CMC4 : pair of "CM C4" */
|
||||
S_one_CMC4,
|
||||
S_odd_CMC4,
|
||||
S_one_C4_odd_CMC4,
|
||||
S_even_CMC4,
|
||||
S_one_C4_even_CMC4,
|
||||
|
||||
S_odd_CM_odd_CMC4,
|
||||
S_even_CM_odd_CMC4,
|
||||
|
||||
S_odd_CM_even_CMC4,
|
||||
S_even_CM_even_CMC4,
|
||||
|
||||
/* C4CM : pair of "C4 CM" */
|
||||
S_odd_C4CM,
|
||||
S_one_CM_odd_C4CM,
|
||||
S_even_C4CM,
|
||||
S_one_CM_even_C4CM,
|
||||
|
||||
S_even_CM_odd_C4CM,
|
||||
S_odd_CM_odd_C4CM,
|
||||
S_even_CM_even_C4CM,
|
||||
S_odd_CM_even_C4CM,
|
||||
};
|
||||
|
||||
static UChar*
|
||||
gb18030_left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
||||
{
|
||||
const UChar *p;
|
||||
enum state state = S_START;
|
||||
|
||||
DEBUG_GB18030(("----------------\n"));
|
||||
for (p = s; p >= start; p--) {
|
||||
DEBUG_GB18030(("state %d --(%02x)-->\n", state, *p));
|
||||
switch (state) {
|
||||
case S_START:
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
return (UChar *)s;
|
||||
case C2:
|
||||
state = S_one_C2; /* C2 */
|
||||
break;
|
||||
case C4:
|
||||
state = S_one_C4; /* C4 */
|
||||
break;
|
||||
case CM:
|
||||
state = S_one_CM; /* CM */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_C2: /* C2 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)s;
|
||||
case CM:
|
||||
state = S_odd_CM_one_CX; /* CM C2 */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_C4: /* C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)s;
|
||||
case CM:
|
||||
state = S_one_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_CM: /* CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)s;
|
||||
case C4:
|
||||
state = S_odd_C4CM;
|
||||
break;
|
||||
case CM:
|
||||
state = S_odd_CM_one_CX; /* CM CM */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_odd_CM_one_CX: /* CM C2 */ /* CM CM */ /* CM CM CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 1);
|
||||
case CM:
|
||||
state = S_even_CM_one_CX;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_even_CM_one_CX: /* CM CM C2 */ /* CM CM CM */ /* CM CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)s;
|
||||
case CM:
|
||||
state = S_odd_CM_one_CX;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_one_CMC4: /* CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)(s - 1);
|
||||
case C4:
|
||||
state = S_one_C4_odd_CMC4; /* C4 CM C4 */
|
||||
break;
|
||||
case CM:
|
||||
state = S_even_CM_one_CX; /* CM CM C4 */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_odd_CMC4: /* CM C4 CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)(s - 1);
|
||||
case C4:
|
||||
state = S_one_C4_odd_CMC4;
|
||||
break;
|
||||
case CM:
|
||||
state = S_odd_CM_odd_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_C4_odd_CMC4: /* C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 1);
|
||||
case CM:
|
||||
state = S_even_CMC4; /* CM C4 CM C4 */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_even_CMC4: /* CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)(s - 3);
|
||||
case C4:
|
||||
state = S_one_C4_even_CMC4;
|
||||
break;
|
||||
case CM:
|
||||
state = S_odd_CM_even_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_C4_even_CMC4: /* C4 CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 3);
|
||||
case CM:
|
||||
state = S_odd_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_odd_CM_odd_CMC4: /* CM CM C4 CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 3);
|
||||
case CM:
|
||||
state = S_even_CM_odd_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_even_CM_odd_CMC4: /* CM CM CM C4 CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 1);
|
||||
case CM:
|
||||
state = S_odd_CM_odd_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_odd_CM_even_CMC4: /* CM CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 1);
|
||||
case CM:
|
||||
state = S_even_CM_even_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_even_CM_even_CMC4: /* CM CM CM C4 CM C4 */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 3);
|
||||
case CM:
|
||||
state = S_odd_CM_even_CMC4;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_odd_C4CM: /* C4 CM */ /* C4 CM C4 CM C4 CM*/
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)s;
|
||||
case CM:
|
||||
state = S_one_CM_odd_C4CM; /* CM C4 CM */
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_CM_odd_C4CM: /* CM C4 CM */ /* CM C4 CM C4 CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)(s - 2); /* |CM C4 CM */
|
||||
case C4:
|
||||
state = S_even_C4CM;
|
||||
break;
|
||||
case CM:
|
||||
state = S_even_CM_odd_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_even_C4CM: /* C4 CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 2); /* C4|CM C4 CM */
|
||||
case CM:
|
||||
state = S_one_CM_even_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_one_CM_even_C4CM: /* CM C4 CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
return (UChar *)(s - 0); /*|CM C4 CM C4|CM */
|
||||
case C4:
|
||||
state = S_odd_C4CM;
|
||||
break;
|
||||
case CM:
|
||||
state = S_even_CM_even_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_even_CM_odd_C4CM: /* CM CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 0); /* |CM CM|C4|CM */
|
||||
case CM:
|
||||
state = S_odd_CM_odd_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_odd_CM_odd_C4CM: /* CM CM CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 2); /* |CM CM|CM C4 CM */
|
||||
case CM:
|
||||
state = S_even_CM_odd_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case S_even_CM_even_C4CM: /* CM CM C4 CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 2); /* |CM CM|C4|CM C4 CM */
|
||||
case CM:
|
||||
state = S_odd_CM_even_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case S_odd_CM_even_C4CM: /* CM CM CM C4 CM C4 CM */
|
||||
switch (GB18030_MAP[*p]) {
|
||||
case C1:
|
||||
case C2:
|
||||
case C4:
|
||||
return (UChar *)(s - 0); /* |CM CM|CM C4 CM C4|CM */
|
||||
case CM:
|
||||
state = S_even_CM_even_C4CM;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DEBUG_GB18030(("state %d\n", state));
|
||||
switch (state) {
|
||||
case S_START: return (UChar *)(s - 0);
|
||||
case S_one_C2: return (UChar *)(s - 0);
|
||||
case S_one_C4: return (UChar *)(s - 0);
|
||||
case S_one_CM: return (UChar *)(s - 0);
|
||||
|
||||
case S_odd_CM_one_CX: return (UChar *)(s - 1);
|
||||
case S_even_CM_one_CX: return (UChar *)(s - 0);
|
||||
|
||||
case S_one_CMC4: return (UChar *)(s - 1);
|
||||
case S_odd_CMC4: return (UChar *)(s - 1);
|
||||
case S_one_C4_odd_CMC4: return (UChar *)(s - 1);
|
||||
case S_even_CMC4: return (UChar *)(s - 3);
|
||||
case S_one_C4_even_CMC4: return (UChar *)(s - 3);
|
||||
|
||||
case S_odd_CM_odd_CMC4: return (UChar *)(s - 3);
|
||||
case S_even_CM_odd_CMC4: return (UChar *)(s - 1);
|
||||
|
||||
case S_odd_CM_even_CMC4: return (UChar *)(s - 1);
|
||||
case S_even_CM_even_CMC4: return (UChar *)(s - 3);
|
||||
|
||||
case S_odd_C4CM: return (UChar *)(s - 0);
|
||||
case S_one_CM_odd_C4CM: return (UChar *)(s - 2);
|
||||
case S_even_C4CM: return (UChar *)(s - 2);
|
||||
case S_one_CM_even_C4CM: return (UChar *)(s - 0);
|
||||
|
||||
case S_even_CM_odd_C4CM: return (UChar *)(s - 0);
|
||||
case S_odd_CM_odd_C4CM: return (UChar *)(s - 2);
|
||||
case S_even_CM_even_C4CM: return (UChar *)(s - 2);
|
||||
case S_odd_CM_even_C4CM: return (UChar *)(s - 0);
|
||||
}
|
||||
|
||||
return (UChar* )s; /* never come here. (escape warning) */
|
||||
}
|
||||
|
||||
static int
|
||||
gb18030_is_allowed_reverse_match(const UChar* s, const UChar* end ARG_UNUSED, OnigEncoding enc)
|
||||
{
|
||||
return GB18030_MAP[*s] == C1 ? TRUE : FALSE;
|
||||
}
|
||||
|
||||
OnigEncodingDefine(gb18030, GB18030) = {
|
||||
gb18030_mbc_enc_len,
|
||||
"GB18030", /* name */
|
||||
4, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
gb18030_mbc_to_code,
|
||||
onigenc_mb4_code_to_mbclen,
|
||||
gb18030_code_to_mbc,
|
||||
gb18030_mbc_case_fold,
|
||||
onigenc_ascii_apply_all_case_fold,
|
||||
onigenc_ascii_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
gb18030_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
gb18030_left_adjust_char_head,
|
||||
gb18030_is_allowed_reverse_match
|
||||
};
|
||||
ENC_REPLICATE("GBK", "GB18030");
|
||||
ENC_ALIAS("CP936", "GBK");
|
||||
ENC_ALIAS("Windows-936", "GBK");
|
|
@ -238,5 +238,5 @@ OnigEncodingDefine(iso_8859_2, ISO_8859_2) = {
|
|||
onigenc_always_true_is_allowed_reverse_match
|
||||
};
|
||||
ENC_ALIAS("ISO8859-2", "ISO-8859-2");
|
||||
ENC_REPLICATE("Windowws-1250", "ISO-8859-2");
|
||||
ENC_REPLICATE("Windows-1250", "ISO-8859-2");
|
||||
ENC_ALIAS("CP1250", "Windows-1250");
|
||||
|
|
|
@ -228,5 +228,3 @@ OnigEncodingDefine(iso_8859_5, ISO_8859_5) = {
|
|||
onigenc_always_true_is_allowed_reverse_match
|
||||
};
|
||||
ENC_ALIAS("ISO8859-5", "ISO-8859-5");
|
||||
ENC_REPLICATE("Windows-1251", "ISO-8859-5");
|
||||
ENC_ALIAS("CP1251", "Windows-1251");
|
||||
|
|
212
enc/koi8_r.c
Normal file
212
enc/koi8_r.c
Normal file
|
@ -0,0 +1,212 @@
|
|||
/**********************************************************************
|
||||
koi8_r.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
#define ENC_KOI8_R_TO_LOWER_CASE(c) EncKOI8_R_ToLowerCaseTable[c]
|
||||
#define ENC_IS_KOI8_R_CTYPE(code,ctype) \
|
||||
((EncKOI8_R_CtypeTable[code] & CTYPE_TO_BIT(ctype)) != 0)
|
||||
|
||||
static const UChar EncKOI8_R_ToLowerCaseTable[256] = {
|
||||
'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
|
||||
'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
|
||||
'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
|
||||
'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
|
||||
'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
|
||||
'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
|
||||
'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
|
||||
'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
|
||||
'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
|
||||
'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
|
||||
'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
|
||||
'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
|
||||
'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
|
||||
'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
|
||||
'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
|
||||
'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
|
||||
'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
|
||||
'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
|
||||
'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
|
||||
'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
|
||||
'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
|
||||
'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
|
||||
'\260', '\261', '\262', '\243', '\264', '\265', '\266', '\267',
|
||||
'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
|
||||
'\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
|
||||
'\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
|
||||
'\320', '\321', '\322', '\323', '\324', '\325', '\326', '\327',
|
||||
'\330', '\331', '\332', '\333', '\334', '\335', '\336', '\337',
|
||||
'\300', '\301', '\302', '\303', '\304', '\305', '\306', '\307',
|
||||
'\310', '\311', '\312', '\313', '\314', '\315', '\316', '\317',
|
||||
'\320', '\321', '\322', '\323', '\324', '\325', '\326', '\327',
|
||||
'\330', '\331', '\332', '\333', '\334', '\335', '\336', '\337'
|
||||
};
|
||||
|
||||
static const unsigned short EncKOI8_R_CtypeTable[256] = {
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4008, 0x420c, 0x4209, 0x4208, 0x4208, 0x4208, 0x4008, 0x4008,
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4284, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0,
|
||||
0x78b0, 0x78b0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x41a0, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x51a0,
|
||||
0x41a0, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x4008,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x0284, 0x00a0, 0x00a0, 0x10a0, 0x01a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x30e2, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x34a2, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2
|
||||
};
|
||||
|
||||
static int
|
||||
koi8_r_mbc_case_fold(OnigCaseFoldType flag ARG_UNUSED,
|
||||
const UChar** pp, const UChar* end ARG_UNUSED, UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
const UChar* p = *pp;
|
||||
|
||||
*lower = ENC_KOI8_R_TO_LOWER_CASE(*p);
|
||||
(*pp)++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int
|
||||
koi8_r_is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end, OnigEncoding enc)
|
||||
{
|
||||
int v;
|
||||
const UChar* p = *pp;
|
||||
|
||||
(*pp)++;
|
||||
v = (EncKOI8_R_CtypeTable[*p] & (BIT_CTYPE_UPPER | BIT_CTYPE_LOWER));
|
||||
return (v != 0 ? TRUE : FALSE);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int
|
||||
koi8_r_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
if (code < 256)
|
||||
return ENC_IS_KOI8_R_CTYPE(code, ctype);
|
||||
else
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
static const OnigPairCaseFoldCodes CaseFoldMap[] = {
|
||||
{ 0xa3, 0xb3 },
|
||||
|
||||
{ 0xc0, 0xe0 },
|
||||
{ 0xc1, 0xe1 },
|
||||
{ 0xc2, 0xe2 },
|
||||
{ 0xc3, 0xe3 },
|
||||
{ 0xc4, 0xe4 },
|
||||
{ 0xc5, 0xe5 },
|
||||
{ 0xc6, 0xe6 },
|
||||
{ 0xc7, 0xe7 },
|
||||
{ 0xc8, 0xe8 },
|
||||
{ 0xc9, 0xe9 },
|
||||
{ 0xca, 0xea },
|
||||
{ 0xcb, 0xeb },
|
||||
{ 0xcc, 0xec },
|
||||
{ 0xcd, 0xed },
|
||||
{ 0xce, 0xee },
|
||||
{ 0xcf, 0xef },
|
||||
|
||||
{ 0xd0, 0xf0 },
|
||||
{ 0xd1, 0xf1 },
|
||||
{ 0xd2, 0xf2 },
|
||||
{ 0xd3, 0xf3 },
|
||||
{ 0xd4, 0xf4 },
|
||||
{ 0xd5, 0xf5 },
|
||||
{ 0xd6, 0xf6 },
|
||||
{ 0xd7, 0xf7 },
|
||||
{ 0xd8, 0xf8 },
|
||||
{ 0xd9, 0xf9 },
|
||||
{ 0xda, 0xfa },
|
||||
{ 0xdb, 0xfb },
|
||||
{ 0xdc, 0xfc },
|
||||
{ 0xdd, 0xfd },
|
||||
{ 0xde, 0xfe },
|
||||
{ 0xdf, 0xff }
|
||||
};
|
||||
|
||||
static int
|
||||
koi8_r_apply_all_case_fold(OnigCaseFoldType flag,
|
||||
OnigApplyAllCaseFoldFunc f, void* arg, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_apply_all_case_fold_with_map(
|
||||
sizeof(CaseFoldMap)/sizeof(OnigPairCaseFoldCodes), CaseFoldMap, 0,
|
||||
flag, f, arg);
|
||||
}
|
||||
|
||||
static int
|
||||
koi8_r_get_case_fold_codes_by_str(OnigCaseFoldType flag,
|
||||
const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[], OnigEncoding enc)
|
||||
{
|
||||
return onigenc_get_case_fold_codes_by_str_with_map(
|
||||
sizeof(CaseFoldMap)/sizeof(OnigPairCaseFoldCodes), CaseFoldMap, 0,
|
||||
flag, p, end, items);
|
||||
}
|
||||
|
||||
OnigEncodingDefine(koi8_r, KOI8_R) = {
|
||||
onigenc_single_byte_mbc_enc_len,
|
||||
"KOI8-R", /* name */
|
||||
1, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
onigenc_single_byte_mbc_to_code,
|
||||
onigenc_single_byte_code_to_mbclen,
|
||||
onigenc_single_byte_code_to_mbc,
|
||||
koi8_r_mbc_case_fold,
|
||||
koi8_r_apply_all_case_fold,
|
||||
koi8_r_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
koi8_r_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
onigenc_single_byte_left_adjust_char_head,
|
||||
onigenc_always_true_is_allowed_reverse_match
|
||||
};
|
|
@ -152,7 +152,7 @@ mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc)
|
|||
int c, i, len;
|
||||
OnigCodePoint n;
|
||||
|
||||
len = enclen(&encoding_Shift_JIS, p, end);
|
||||
len = enclen(&OnigEncodingName(Shift_JIS), p, end);
|
||||
c = *p++;
|
||||
n = c;
|
||||
if (len == 1) return n;
|
||||
|
@ -174,7 +174,7 @@ code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc)
|
|||
*p++ = (UChar )(code & 0xff);
|
||||
|
||||
#if 0
|
||||
if (enclen(&encoding_Shift_JIS, buf) != (p - buf))
|
||||
if (enclen(&OnigEncodingName(Shift_JIS), buf) != (p - buf))
|
||||
return REGERR_INVALID_CODE_POINT_VALUE;
|
||||
#endif
|
||||
return p - buf;
|
||||
|
@ -194,7 +194,7 @@ mbc_case_fold(OnigCaseFoldType flag,
|
|||
}
|
||||
else {
|
||||
int i;
|
||||
int len = enclen(&encoding_Shift_JIS, p, end);
|
||||
int len = enclen(&OnigEncodingName(Shift_JIS), p, end);
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
*lower++ = *p++;
|
||||
|
@ -209,7 +209,7 @@ static int
|
|||
is_mbc_ambiguous(OnigCaseFoldType flag,
|
||||
const UChar** pp, const UChar* end)
|
||||
{
|
||||
return onigenc_mbn_is_mbc_ambiguous(&encoding_Shift_JIS, flag, pp, end);
|
||||
return onigenc_mbn_is_mbc_ambiguous(&OnigEncodingName(Shift_JIS), flag, pp, end);
|
||||
|
||||
}
|
||||
#endif
|
||||
|
@ -247,7 +247,7 @@ left_adjust_char_head(const UChar* start, const UChar* s, OnigEncoding enc)
|
|||
}
|
||||
}
|
||||
}
|
||||
len = enclen(&encoding_Shift_JIS, p, s);
|
||||
len = enclen(&OnigEncodingName(Shift_JIS), p, s);
|
||||
if (p + len > s) return (UChar* )p;
|
||||
p += len;
|
||||
return (UChar* )(p + ((s - p) & ~1));
|
||||
|
|
201
enc/windows_1251.c
Normal file
201
enc/windows_1251.c
Normal file
|
@ -0,0 +1,201 @@
|
|||
/**********************************************************************
|
||||
cp1251.c - Oniguruma (regular expression library)
|
||||
**********************************************************************/
|
||||
/*-
|
||||
* Copyright (c) 2006-2007 Byte <byte AT mail DOT kna DOT ru>
|
||||
* K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "regenc.h"
|
||||
|
||||
#define ENC_CP1251_TO_LOWER_CASE(c) EncCP1251_ToLowerCaseTable[c]
|
||||
#define ENC_IS_CP1251_CTYPE(code,ctype) \
|
||||
((EncCP1251_CtypeTable[code] & CTYPE_TO_BIT(ctype)) != 0)
|
||||
|
||||
static const UChar EncCP1251_ToLowerCaseTable[256] = {
|
||||
'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
|
||||
'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
|
||||
'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
|
||||
'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
|
||||
'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
|
||||
'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
|
||||
'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
|
||||
'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
|
||||
'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
|
||||
'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
|
||||
'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
|
||||
'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
|
||||
'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
|
||||
'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
|
||||
'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
|
||||
'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
|
||||
'\220', '\203', '\202', '\203', '\204', '\205', '\206', '\207',
|
||||
'\210', '\211', '\232', '\213', '\234', '\235', '\236', '\237',
|
||||
'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
|
||||
'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
|
||||
'\240', '\242', '\242', '\274', '\244', '\264', '\246', '\247',
|
||||
'\270', '\251', '\272', '\253', '\254', '\255', '\256', '\277',
|
||||
'\260', '\261', '\263', '\263', '\264', '\265', '\266', '\267',
|
||||
'\270', '\271', '\272', '\273', '\274', '\276', '\276', '\277',
|
||||
'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
|
||||
'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
|
||||
'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
|
||||
'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
|
||||
'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
|
||||
'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
|
||||
'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
|
||||
'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377'
|
||||
};
|
||||
|
||||
static const unsigned short EncCP1251_CtypeTable[256] = {
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4008, 0x428c, 0x4209, 0x4208, 0x4208, 0x4208, 0x4008, 0x4008,
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
|
||||
0x4284, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0,
|
||||
0x78b0, 0x78b0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
|
||||
0x41a0, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
|
||||
0x74a2, 0x74a2, 0x74a2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x51a0,
|
||||
0x41a0, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
|
||||
0x70e2, 0x70e2, 0x70e2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x4008,
|
||||
0x34a2, 0x34a2, 0x01a0, 0x30e2, 0x01a0, 0x01a0, 0x01a0, 0x01a0,
|
||||
0x0000, 0x01a0, 0x34a2, 0x01a0, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x30e2, 0x01a0, 0x01a0, 0x01a0, 0x01a0, 0x01a0, 0x01a0, 0x01a0,
|
||||
0x0008, 0x0000, 0x30e2, 0x01a0, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x0280, 0x34a2, 0x30e2, 0x34a2, 0x01a0, 0x34a2, 0x01a0, 0x01a0,
|
||||
0x34a2, 0x01a0, 0x34a2, 0x01a0, 0x01a0, 0x01a0, 0x01a0, 0x34a2,
|
||||
0x01a0, 0x01a0, 0x34a2, 0x30e2, 0x30e2, 0x31e2, 0x01a0, 0x01a0,
|
||||
0x30e2, 0x0000, 0x30e2, 0x01a0, 0x30e2, 0x34a2, 0x30e2, 0x30e2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
|
||||
0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2
|
||||
};
|
||||
|
||||
static int
|
||||
cp1251_mbc_case_fold(OnigCaseFoldType flag ARG_UNUSED,
|
||||
const UChar** pp, const UChar* end ARG_UNUSED, UChar* lower, OnigEncoding enc)
|
||||
{
|
||||
const UChar* p = *pp;
|
||||
|
||||
*lower = ENC_CP1251_TO_LOWER_CASE(*p);
|
||||
(*pp)++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int
|
||||
cp1251_is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc)
|
||||
{
|
||||
if (code < 256)
|
||||
return ENC_IS_CP1251_CTYPE(code, ctype);
|
||||
else
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
static const OnigPairCaseFoldCodes CaseFoldMap[] = {
|
||||
{ 0xb8, 0xa8 },
|
||||
|
||||
{ 0xe0, 0xc0 },
|
||||
{ 0xe1, 0xc1 },
|
||||
{ 0xe2, 0xc2 },
|
||||
{ 0xe3, 0xc3 },
|
||||
{ 0xe4, 0xc4 },
|
||||
{ 0xe5, 0xc5 },
|
||||
{ 0xe6, 0xc6 },
|
||||
{ 0xe7, 0xc7 },
|
||||
{ 0xe8, 0xc8 },
|
||||
{ 0xe9, 0xc9 },
|
||||
{ 0xea, 0xca },
|
||||
{ 0xeb, 0xcb },
|
||||
{ 0xec, 0xcc },
|
||||
{ 0xed, 0xcd },
|
||||
{ 0xee, 0xce },
|
||||
{ 0xef, 0xcf },
|
||||
|
||||
{ 0xf0, 0xd0 },
|
||||
{ 0xf1, 0xd1 },
|
||||
{ 0xf2, 0xd2 },
|
||||
{ 0xf3, 0xd3 },
|
||||
{ 0xf4, 0xd4 },
|
||||
{ 0xf5, 0xd5 },
|
||||
{ 0xf6, 0xd6 },
|
||||
{ 0xf7, 0xd7 },
|
||||
{ 0xf8, 0xd8 },
|
||||
{ 0xf9, 0xd9 },
|
||||
{ 0xfa, 0xda },
|
||||
{ 0xfb, 0xdb },
|
||||
{ 0xfc, 0xdc },
|
||||
{ 0xfd, 0xdd },
|
||||
{ 0xfe, 0xde },
|
||||
{ 0xff, 0xdf }
|
||||
};
|
||||
|
||||
static int
|
||||
cp1251_apply_all_case_fold(OnigCaseFoldType flag,
|
||||
OnigApplyAllCaseFoldFunc f, void* arg, OnigEncoding enc)
|
||||
{
|
||||
return onigenc_apply_all_case_fold_with_map(
|
||||
sizeof(CaseFoldMap)/sizeof(OnigPairCaseFoldCodes), CaseFoldMap, 0,
|
||||
flag, f, arg);
|
||||
}
|
||||
|
||||
static int
|
||||
cp1251_get_case_fold_codes_by_str(OnigCaseFoldType flag,
|
||||
const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[], OnigEncoding enc)
|
||||
{
|
||||
return onigenc_get_case_fold_codes_by_str_with_map(
|
||||
sizeof(CaseFoldMap)/sizeof(OnigPairCaseFoldCodes), CaseFoldMap, 0,
|
||||
flag, p, end, items);
|
||||
}
|
||||
|
||||
OnigEncodingType OnigEncodingCP1251 = {
|
||||
onigenc_single_byte_mbc_enc_len,
|
||||
"Windows-1251", /* name */
|
||||
1, /* max enc length */
|
||||
1, /* min enc length */
|
||||
onigenc_is_mbc_newline_0x0a,
|
||||
onigenc_single_byte_mbc_to_code,
|
||||
onigenc_single_byte_code_to_mbclen,
|
||||
onigenc_single_byte_code_to_mbc,
|
||||
cp1251_mbc_case_fold,
|
||||
cp1251_apply_all_case_fold,
|
||||
cp1251_get_case_fold_codes_by_str,
|
||||
onigenc_minimum_property_name_to_ctype,
|
||||
cp1251_is_code_ctype,
|
||||
onigenc_not_support_get_ctype_code_range,
|
||||
onigenc_single_byte_left_adjust_char_head,
|
||||
onigenc_always_true_is_allowed_reverse_match
|
||||
};
|
||||
ENC_ALIAS("CP1251", "Windows-1251");
|
|
@ -173,68 +173,10 @@ typedef struct OnigEncodingTypeST {
|
|||
typedef OnigEncodingType* OnigEncoding;
|
||||
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingASCII;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_1;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_2;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_3;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_4;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_5;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_6;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_7;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_8;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_9;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_10;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_11;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_13;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_14;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_15;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingISO_8859_16;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingUTF8;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingUTF_16BE;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingUTF_16LE;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingUTF_32BE;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingUTF_32LE;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingEUC_JP;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingEUC_TW;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingEUC_KR;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingEUC_CN;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingSJIS;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingKOI8;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingKOI8_R;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingCP1251;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingBIG5;
|
||||
ONIG_EXTERN OnigEncodingType OnigEncodingGB18030;
|
||||
|
||||
#define ONIG_ENCODING_ASCII (&OnigEncodingASCII)
|
||||
#define ONIG_ENCODING_ISO_8859_1 (&OnigEncodingISO_8859_1)
|
||||
#define ONIG_ENCODING_ISO_8859_2 (&OnigEncodingISO_8859_2)
|
||||
#define ONIG_ENCODING_ISO_8859_3 (&OnigEncodingISO_8859_3)
|
||||
#define ONIG_ENCODING_ISO_8859_4 (&OnigEncodingISO_8859_4)
|
||||
#define ONIG_ENCODING_ISO_8859_5 (&OnigEncodingISO_8859_5)
|
||||
#define ONIG_ENCODING_ISO_8859_6 (&OnigEncodingISO_8859_6)
|
||||
#define ONIG_ENCODING_ISO_8859_7 (&OnigEncodingISO_8859_7)
|
||||
#define ONIG_ENCODING_ISO_8859_8 (&OnigEncodingISO_8859_8)
|
||||
#define ONIG_ENCODING_ISO_8859_9 (&OnigEncodingISO_8859_9)
|
||||
#define ONIG_ENCODING_ISO_8859_10 (&OnigEncodingISO_8859_10)
|
||||
#define ONIG_ENCODING_ISO_8859_11 (&OnigEncodingISO_8859_11)
|
||||
#define ONIG_ENCODING_ISO_8859_13 (&OnigEncodingISO_8859_13)
|
||||
#define ONIG_ENCODING_ISO_8859_14 (&OnigEncodingISO_8859_14)
|
||||
#define ONIG_ENCODING_ISO_8859_15 (&OnigEncodingISO_8859_15)
|
||||
#define ONIG_ENCODING_ISO_8859_16 (&OnigEncodingISO_8859_16)
|
||||
#define ONIG_ENCODING_UTF8 (&OnigEncodingUTF8)
|
||||
#define ONIG_ENCODING_UTF_16BE (&OnigEncodingUTF_16BE)
|
||||
#define ONIG_ENCODING_UTF_16LE (&OnigEncodingUTF_16LE)
|
||||
#define ONIG_ENCODING_UTF_32BE (&OnigEncodingUTF_32BE)
|
||||
#define ONIG_ENCODING_UTF_32LE (&OnigEncodingUTF_32LE)
|
||||
#define ONIG_ENCODING_EUC_JP (&OnigEncodingEUC_JP)
|
||||
#define ONIG_ENCODING_EUC_TW (&OnigEncodingEUC_TW)
|
||||
#define ONIG_ENCODING_EUC_KR (&OnigEncodingEUC_KR)
|
||||
#define ONIG_ENCODING_EUC_CN (&OnigEncodingEUC_CN)
|
||||
#define ONIG_ENCODING_SJIS (&OnigEncodingSJIS)
|
||||
#define ONIG_ENCODING_KOI8 (&OnigEncodingKOI8)
|
||||
#define ONIG_ENCODING_KOI8_R (&OnigEncodingKOI8_R)
|
||||
#define ONIG_ENCODING_CP1251 (&OnigEncodingCP1251)
|
||||
#define ONIG_ENCODING_BIG5 (&OnigEncodingBIG5)
|
||||
#define ONIG_ENCODING_GB18030 (&OnigEncodingGB18030)
|
||||
|
||||
#define ONIG_ENCODING_UNDEF ((OnigEncoding )0)
|
||||
|
||||
|
|
4
regenc.c
4
regenc.c
|
@ -712,14 +712,14 @@ onigenc_mbn_is_mbc_ambiguous(OnigEncoding enc, OnigCaseFoldType flag,
|
|||
#endif
|
||||
|
||||
extern int
|
||||
onigenc_mb2_code_to_mbclen(OnigCodePoint code)
|
||||
onigenc_mb2_code_to_mbclen(OnigCodePoint code, OnigEncoding enc)
|
||||
{
|
||||
if ((code & 0xff00) != 0) return 2;
|
||||
else return 1;
|
||||
}
|
||||
|
||||
extern int
|
||||
onigenc_mb4_code_to_mbclen(OnigCodePoint code)
|
||||
onigenc_mb4_code_to_mbclen(OnigCodePoint code, OnigEncoding enc)
|
||||
{
|
||||
if ((code & 0xff000000) != 0) return 4;
|
||||
else if ((code & 0xff0000) != 0) return 3;
|
||||
|
|
4
regenc.h
4
regenc.h
|
@ -133,12 +133,12 @@ ONIG_EXTERN int onigenc_ascii_is_code_ctype P_((OnigCodePoint code, unsigned int
|
|||
/* methods for multi byte encoding */
|
||||
ONIG_EXTERN OnigCodePoint onigenc_mbn_mbc_to_code P_((OnigEncoding enc, const UChar* p, const UChar* end));
|
||||
ONIG_EXTERN int onigenc_mbn_mbc_case_fold P_((OnigEncoding enc, OnigCaseFoldType flag, const UChar** p, const UChar* end, UChar* lower));
|
||||
ONIG_EXTERN int onigenc_mb2_code_to_mbclen P_((OnigCodePoint code));
|
||||
ONIG_EXTERN int onigenc_mb2_code_to_mbclen P_((OnigCodePoint code, OnigEncoding enc));
|
||||
ONIG_EXTERN int onigenc_mb2_code_to_mbc P_((OnigEncoding enc, OnigCodePoint code, UChar *buf));
|
||||
ONIG_EXTERN int onigenc_minimum_property_name_to_ctype P_((OnigEncoding enc, UChar* p, UChar* end));
|
||||
ONIG_EXTERN int onigenc_unicode_property_name_to_ctype P_((OnigEncoding enc, UChar* p, UChar* end));
|
||||
ONIG_EXTERN int onigenc_mb2_is_code_ctype P_((OnigEncoding enc, OnigCodePoint code, unsigned int ctype));
|
||||
ONIG_EXTERN int onigenc_mb4_code_to_mbclen P_((OnigCodePoint code));
|
||||
ONIG_EXTERN int onigenc_mb4_code_to_mbclen P_((OnigCodePoint code, OnigEncoding enc));
|
||||
ONIG_EXTERN int onigenc_mb4_code_to_mbc P_((OnigEncoding enc, OnigCodePoint code, UChar *buf));
|
||||
ONIG_EXTERN int onigenc_mb4_is_code_ctype P_((OnigEncoding enc, OnigCodePoint code, unsigned int ctype));
|
||||
|
||||
|
|
Loading…
Reference in a new issue