kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

* tool/transcode-tblgen.rb (transcode_compile_tree): make

Browse source 
valid_encoding mandatory unless from_encoding is registered in
  ValidEncoding.
  (transcode_tbl_only): ditto.
  (transcode_tblgen): ditto.
  (ValidEncoding): new function.

* enc/trans/escape.trans: specify valid_encoding.

* enc/trans/emoji_sjis_docomo.trans: ditto.

* enc/trans/emoji.trans: ditto.

* enc/trans/emoji_iso2022_kddi.trans: ditto.

* enc/trans/big5.trans: ditto.

* enc/trans/emoji_sjis_softbank.trans: ditto.

* enc/trans/emoji_sjis_kddi.trans: ditto.

* enc/trans/chinese.trans: use ValidEncoding() instead of
  ValidEncoding[].


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@26995 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
akr 2010-03-21 03:38:58 +00:00
parent f623979f65
commit 49d993729f
10 changed files with 63 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
ChangeLog
View file

@ -1,3 +1,29 @@
Sun Mar 21 12:32:39 2010 Tanaka Akira <akr@fsij.org>
* tool/transcode-tblgen.rb (transcode_compile_tree): make
valid_encoding mandatory unless from_encoding is registered in
ValidEncoding.
(transcode_tbl_only): ditto.
(transcode_tblgen): ditto.
(ValidEncoding): new function.
* enc/trans/escape.trans: specify valid_encoding.
* enc/trans/emoji_sjis_docomo.trans: ditto.
* enc/trans/emoji.trans: ditto.
* enc/trans/emoji_iso2022_kddi.trans: ditto.
* enc/trans/big5.trans: ditto.
* enc/trans/emoji_sjis_softbank.trans: ditto.
* enc/trans/emoji_sjis_kddi.trans: ditto.
* enc/trans/chinese.trans: use ValidEncoding() instead of
ValidEncoding[].
Sun Mar 21 09:43:01 2010 Nobuyoshi Nakada <nobu@ruby-lang.org>
* io.c (IO_[CRW]BUF_CAPA_MIN): replaced magic numbers.

6
enc/trans/big5.trans
View file

@ -8,10 +8,10 @@
transcode_tblgen "Big5", "UTF-8", [["{00-7f}", :nomap], *BIG5_TO_UCS_TBL]
transcode_tblgen "UTF-8", "Big5", [["{00-7f}", :nomap], *BIG5_TO_UCS_TBL.map {|a,b| [b,a] }]
transcode_tblgen "Big5-HKSCS", "UTF-8", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL]
transcode_tblgen "Big5-HKSCS", "UTF-8", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL], ValidEncoding('Big5')
transcode_tblgen "UTF-8", "Big5-HKSCS", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL.map {|a,b| [b,a] }]
transcode_tblgen "Big5-UAO", "UTF-8", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL]
transcode_tblgen "Big5-UAO", "UTF-8", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL], ValidEncoding('Big5')
transcode_tblgen "UTF-8", "Big5-UAO", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL.map {|a,b| [b,a] }]
%>
@ -22,4 +22,4 @@ void
Init_big5(void)
{
<%= transcode_register_code %>
}
}

4
enc/trans/chinese.trans
View file

@ -1,8 +1,8 @@
#include "transcode_data.h"
<%
gb2312_valid_byte_pattern = ValidEncoding['EUC-KR']
gb12345_valid_byte_pattern = ValidEncoding['EUC-KR']
gb2312_valid_byte_pattern = ValidEncoding('EUC-KR')
gb12345_valid_byte_pattern = ValidEncoding('EUC-KR')
transcode_tblgen "GB2312", "UTF-8",
[["{00-7f}", :nomap]] +

2
enc/trans/emoji.trans
View file

@ -23,7 +23,7 @@
to_encoding = (to_company == "Unicode") ? "UTF-8" : "UTF8-"+to_company
strict_mapping = EMOJI_EXCHANGE_TBL[from_encoding][to_encoding]
transcode_tblgen(from_encoding, to_encoding, strict_mapping + nomap_table)
transcode_tblgen(from_encoding, to_encoding, strict_mapping + nomap_table, nil)
end
end
%>

6
enc/trans/emoji_iso2022_kddi.trans
View file

@ -32,7 +32,8 @@
CP932VDC:NEC_IBM/UCS
CP932VDC:IBM/UCS
CP932UDA/UCS
JISX0208:MS/UCS).join(","))
JISX0208:MS/UCS).join(",")),
nil
# mapping from UTF8-KDDI to stateless-ISO-2022-JP-KDDI
transcode_tblgen "UTF8-KDDI", "stateless-ISO-2022-JP-KDDI",
@ -43,7 +44,8 @@
UCS/CP932VDC:NEC_IBM
UCS/CP932VDC:IBM
UCS/CP932UDA
UCS/JISX0208:MS).join(","))
UCS/JISX0208:MS).join(",")),
ValidEncoding('UTF-8')
%>
<%= transcode_generated_code %>

6
enc/trans/emoji_sjis_docomo.trans
View file

@ -9,7 +9,8 @@
CP932VDC:NEC_IBM/UCS
CP932VDC:IBM/UCS
CP932UDA/UCS
JISX0208:MS/UCS).join(","))
JISX0208:MS/UCS).join(",")),
ValidEncoding('Shift_JIS')
transcode_tblgen "UTF8-DOCOMO", "SJIS-DOCOMO",
[["{00-7f}", :nomap]] +
@ -19,7 +20,8 @@
UCS/CP932VDC:NEC_IBM
UCS/CP932VDC:IBM
UCS/CP932UDA
UCS/JISX0208:MS).join(","))
UCS/JISX0208:MS).join(",")),
ValidEncoding('UTF-8')
%>
<%= transcode_generated_code %>

6
enc/trans/emoji_sjis_kddi.trans
View file

@ -9,7 +9,8 @@
CP932VDC:NEC_IBM/UCS
CP932VDC:IBM/UCS
CP932UDA/UCS
JISX0208:MS/UCS).join(","))
JISX0208:MS/UCS).join(",")),
ValidEncoding('Shift_JIS')
transcode_tblgen "UTF8-KDDI", "SJIS-KDDI",
[["{00-7f}", :nomap]] +
@ -20,7 +21,8 @@
UCS/CP932VDC:NEC_IBM
UCS/CP932VDC:IBM
UCS/CP932UDA
UCS/JISX0208:MS).join(","))
UCS/JISX0208:MS).join(",")),
ValidEncoding('UTF-8')
%>
<%= transcode_generated_code %>

6
enc/trans/emoji_sjis_softbank.trans
View file

@ -9,7 +9,8 @@
CP932VDC:NEC_IBM/UCS
CP932VDC:IBM/UCS
CP932UDA/UCS
JISX0208:MS/UCS).join(","))
JISX0208:MS/UCS).join(",")),
ValidEncoding('Shift_JIS')
transcode_tblgen "UTF8-SoftBank", "SJIS-SoftBank",
[["{00-7f}", :nomap]] +
@ -19,7 +20,8 @@
UCS/CP932VDC:NEC_IBM
UCS/CP932VDC:IBM
UCS/CP932UDA
UCS/JISX0208:MS).join(","))
UCS/JISX0208:MS).join(",")),
ValidEncoding('UTF-8')
%>
<%= transcode_generated_code %>

6
enc/trans/escape.trans
View file

@ -8,14 +8,14 @@
transcode_tblgen("", "amp_escape", [
["{00-25,27-FF}", :nomap],
["26", hexstr("&amp;")]
])
], nil)
transcode_tblgen("", "xml_text_escape", [
["{00-25,27-3B,3D,3F-FF}", :nomap],
["26", hexstr("&amp;")],
["3C", hexstr("&lt;")],
["3E", hexstr("&gt;")]
])
], nil)
transcode_tblgen("", "xml_attr_content_escape", [
["{00-21,23-25,27-3B,3D,3F-FF}", :nomap],
@ -23,7 +23,7 @@
["26", hexstr("&amp;")],
["3C", hexstr("&lt;")],
["3E", hexstr("&gt;")]
])
], nil)
map_xml_attr_quote = {}
map_xml_attr_quote["{00-FF}"] = :func_so

16
tool/transcode-tblgen.rb
View file

@ -758,13 +758,17 @@ def encode_utf8(map)
r
end
def transcode_compile_tree(name, from, map, valid_encoding=nil)
UnspecifiedValidEncoding = Object.new
def transcode_compile_tree(name, from, map, valid_encoding)
map = encode_utf8(map)
h = {}
map.each {|k, v|
h[k] = v unless h[k] # use first mapping
}
valid_encoding = ValidEncoding[from] if valid_encoding == nil
if valid_encoding.equal? UnspecifiedValidEncoding
valid_encoding = ValidEncoding.fetch(from)
end
if valid_encoding
am = ActionMap.merge2(h, {valid_encoding => :undef}) {|prefix, as1, as2|
a1 = as1.empty? ? nil : ActionMap.unambiguous_action(as1)
@ -787,7 +791,7 @@ end
TRANSCODERS = []
TRANSCODE_GENERATED_TRANSCODER_CODE = ''
def transcode_tbl_only(from, to, map, valid_encoding=nil)
def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding)
if VERBOSE_MODE
if from.empty? || to.empty?
STDERR.puts "converter for #{from.empty? ? to : from}"
@ -808,7 +812,7 @@ def transcode_tbl_only(from, to, map, valid_encoding=nil)
return map, tree_name, real_tree_name, max_input
end
def transcode_tblgen(from, to, map, valid_encoding=nil)
def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding)
map, tree_name, real_tree_name, max_input = transcode_tbl_only(from, to, map, valid_encoding)
transcoder_name = "rb_#{tree_name}"
TRANSCODERS << transcoder_name
@ -912,6 +916,10 @@ ValidEncoding = {
{81-fe}{30-39}{81-fe}{30-39}',
}
def ValidEncoding(enc)
ValidEncoding.fetch(enc)
end
def set_valid_byte_pattern(encoding, pattern_or_label)
pattern =
if ValidEncoding[pattern_or_label]