1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00

* lib/uri/common.rb (URI.decode_www_form): follow current URL Standard.

It gets encoding argument to specify the character encoding.
  It now allows loose percent encoded strings, but denies ;-separator.
  [ruby-core:53475] [Bug #8103]

* lib/uri/common.rb (URI.decode_www_form): follow current URL Standard.
  It gets encoding argument to convert before percent encode.
  Now UTF-16 strings aren't converted to UTF-8 before percent encode
  by default.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@40460 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
naruse 2013-04-25 05:27:41 +00:00
parent 5a3fab5918
commit 4a50d447d9
4 changed files with 340 additions and 38 deletions

View file

@ -1,3 +1,15 @@
Thu Apr 25 14:26:32 2013 NARUSE, Yui <naruse@ruby-lang.org>
* lib/uri/common.rb (URI.decode_www_form): follow current URL Standard.
It gets encoding argument to specify the character encoding.
It now allows loose percent encoded strings, but denies ;-separator.
[ruby-core:53475] [Bug #8103]
* lib/uri/common.rb (URI.decode_www_form): follow current URL Standard.
It gets encoding argument to convert before percent encode.
Now UTF-16 strings aren't converted to UTF-8 before percent encode
by default.
Wed Apr 25 14:26:00 2013 Charlie Somerville <charlie@charliesomerville.com>
* benchmark/bm_hash_shift.rb: add benchmark for Hash#shift

10
NEWS
View file

@ -75,4 +75,14 @@ with all sufficient information, see the ChangeLog file.
* Tempfile.create
=== Stdlib compatibility issues (excluding feature bug fixes)
* URI
* incompatible changes:
* URI.decode_www_form follows current WHATWG URL Standard.
It gets encoding argument to specify the character encoding.
It now allows loose percent encoded strings, but denies ;-separator.
* URI.encode_www_form follows current WHATWG URL Standard.
It gets encoding argument to convert before percent encode.
UTF-16 strings aren't converted to UTF-8 before percent encode by default.
=== C API updates

View file

@ -873,18 +873,21 @@ module URI
# This method doesn't convert *, -, ., 0-9, A-Z, _, a-z, but does convert SP
# (ASCII space) to + and converts others to %XX.
#
# If +enc+ is given, convert +str+ to the encoding before percent encoding.
#
# This is an implementation of
# http://www.w3.org/TR/html5/association-of-controls-and-forms.html#url-encoded-form-data
#
# See URI.decode_www_form_component, URI.encode_www_form
def self.encode_www_form_component(str)
str = str.to_s
if HTML5ASCIIINCOMPAT.include?(str.encoding)
str = str.encode(Encoding::UTF_8)
else
str = str.dup
def self.encode_www_form_component(str, enc=nil)
str = str.to_s.dup
if str.encoding != Encoding::ASCII_8BIT
if enc && enc != Encoding::ASCII_8BIT
str.encode!(Encoding::UTF_8, invalid: :replace, undef: :replace)
str.encode!(enc, fallback: ->(x){"&#{x.ord};"})
end
str.force_encoding(Encoding::ASCII_8BIT)
end
str.force_encoding(Encoding::ASCII_8BIT)
str.gsub!(/[^*\-.0-9A-Z_a-z]/, TBLENCWWWCOMP_)
str.force_encoding(Encoding::US_ASCII)
end
@ -914,8 +917,7 @@ module URI
# This method doesn't handle files. When you send a file, use
# multipart/form-data.
#
# This is an implementation of
# http://www.w3.org/TR/html5/forms.html#url-encoded-form-data
# This refers http://url.spec.whatwg.org/#concept-urlencoded-serializer
#
# URI.encode_www_form([["q", "ruby"], ["lang", "en"]])
# #=> "q=ruby&lang=en"
@ -927,39 +929,33 @@ module URI
# #=> "q=ruby&q=perl&lang=en"
#
# See URI.encode_www_form_component, URI.decode_www_form
def self.encode_www_form(enum)
def self.encode_www_form(enum, enc=nil)
enum.map do |k,v|
if v.nil?
encode_www_form_component(k)
encode_www_form_component(k, enc)
elsif v.respond_to?(:to_ary)
v.to_ary.map do |w|
str = encode_www_form_component(k)
str = encode_www_form_component(k, enc)
unless w.nil?
str << '='
str << encode_www_form_component(w)
str << encode_www_form_component(w, enc)
end
end.join('&')
else
str = encode_www_form_component(k)
str = encode_www_form_component(k, enc)
str << '='
str << encode_www_form_component(v)
str << encode_www_form_component(v, enc)
end
end.join('&')
end
WFKV_ = '(?:[^%#=;&]*(?:%\h\h[^%#=;&]*)*)' # :nodoc:
# Decode URL-encoded form data from given +str+.
#
# This decodes application/x-www-form-urlencoded data
# and returns array of key-value array.
# This internally uses URI.decode_www_form_component.
#
# _charset_ hack is not supported now because the mapping from given charset
# to Ruby's encoding is not clear yet.
# see also http://www.w3.org/TR/html5/syntax.html#character-encodings-0
#
# This refers http://www.w3.org/TR/html5/forms.html#url-encoded-form-data
# This refers http://url.spec.whatwg.org/#concept-urlencoded-parser ,
# so this supports only &-separator, don't support ;-separator.
#
# ary = URI.decode_www_form("a=1&a=2&b=3")
# p ary #=> [['a', '1'], ['a', '2'], ['b', '3']]
@ -969,17 +965,263 @@ module URI
# p Hash[ary] # => {"a"=>"2", "b"=>"3"}
#
# See URI.decode_www_form_component, URI.encode_www_form
def self.decode_www_form(str, enc=Encoding::UTF_8)
return [] if str.empty?
unless /\A#{WFKV_}=#{WFKV_}(?:[;&]#{WFKV_}=#{WFKV_})*\z/o =~ str
raise ArgumentError, "invalid data of application/x-www-form-urlencoded (#{str})"
end
def self.decode_www_form(str, enc=Encoding::UTF_8, separator: '&', use__charset_: false, isindex: false)
raise ArgumentError, "the input of #{self.name}.#{__method__} must be ASCII only string" unless str.ascii_only?
ary = []
$&.scan(/([^=;&]+)=([^;&]*)/) do
ary << [decode_www_form_component($1, enc), decode_www_form_component($2, enc)]
return ary if str.empty?
enc = Encoding.find(enc)
str.b.each_line(separator) do |string|
string.chomp!(separator)
key, sep, val = string.partition('=')
if isindex
if sep.empty?
val = key
key = ''
end
isindex = false
end
if use__charset_
if key == '_charset_'
if e = get_encoding(val)
enc = e
use__charset_ = false
ary.each do |k, v|
v.force_encoding(enc)
k.force_encoding(enc)
end
end
end
end
key.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_)
if val
val.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_)
else
val = ''
end
val.force_encoding(enc)
key.force_encoding(enc)
ary << [key, val]
end
ary
end
private
# curl http://encoding.spec.whatwg.org/encodings.json|rb -rpp -rjson -e'H={};h={"shift_jis"=>"Windows-31J","euc-jp"=>"cp51932","iso-2022-jp"=>"cp50221","x-mac-cyrillic"=>"macCyrillic"};JSON($<.read).map{|x|x["encodings"]}.flatten.each{|x|Encoding.find(n=h.fetch(n=x["name"],n))rescue next;x["labels"].each{|y|H[y]=n}};pp H'
WEB_ENCODINGS_ = {
"unicode-1-1-utf-8"=>"utf-8",
"utf-8"=>"utf-8",
"utf8"=>"utf-8",
"866"=>"ibm866",
"cp866"=>"ibm866",
"csibm866"=>"ibm866",
"ibm866"=>"ibm866",
"csisolatin2"=>"iso-8859-2",
"iso-8859-2"=>"iso-8859-2",
"iso-ir-101"=>"iso-8859-2",
"iso8859-2"=>"iso-8859-2",
"iso88592"=>"iso-8859-2",
"iso_8859-2"=>"iso-8859-2",
"iso_8859-2:1987"=>"iso-8859-2",
"l2"=>"iso-8859-2",
"latin2"=>"iso-8859-2",
"csisolatin3"=>"iso-8859-3",
"iso-8859-3"=>"iso-8859-3",
"iso-ir-109"=>"iso-8859-3",
"iso8859-3"=>"iso-8859-3",
"iso88593"=>"iso-8859-3",
"iso_8859-3"=>"iso-8859-3",
"iso_8859-3:1988"=>"iso-8859-3",
"l3"=>"iso-8859-3",
"latin3"=>"iso-8859-3",
"csisolatin4"=>"iso-8859-4",
"iso-8859-4"=>"iso-8859-4",
"iso-ir-110"=>"iso-8859-4",
"iso8859-4"=>"iso-8859-4",
"iso88594"=>"iso-8859-4",
"iso_8859-4"=>"iso-8859-4",
"iso_8859-4:1988"=>"iso-8859-4",
"l4"=>"iso-8859-4",
"latin4"=>"iso-8859-4",
"csisolatincyrillic"=>"iso-8859-5",
"cyrillic"=>"iso-8859-5",
"iso-8859-5"=>"iso-8859-5",
"iso-ir-144"=>"iso-8859-5",
"iso8859-5"=>"iso-8859-5",
"iso88595"=>"iso-8859-5",
"iso_8859-5"=>"iso-8859-5",
"iso_8859-5:1988"=>"iso-8859-5",
"arabic"=>"iso-8859-6",
"asmo-708"=>"iso-8859-6",
"csiso88596e"=>"iso-8859-6",
"csiso88596i"=>"iso-8859-6",
"csisolatinarabic"=>"iso-8859-6",
"ecma-114"=>"iso-8859-6",
"iso-8859-6"=>"iso-8859-6",
"iso-8859-6-e"=>"iso-8859-6",
"iso-8859-6-i"=>"iso-8859-6",
"iso-ir-127"=>"iso-8859-6",
"iso8859-6"=>"iso-8859-6",
"iso88596"=>"iso-8859-6",
"iso_8859-6"=>"iso-8859-6",
"iso_8859-6:1987"=>"iso-8859-6",
"csisolatingreek"=>"iso-8859-7",
"ecma-118"=>"iso-8859-7",
"elot_928"=>"iso-8859-7",
"greek"=>"iso-8859-7",
"greek8"=>"iso-8859-7",
"iso-8859-7"=>"iso-8859-7",
"iso-ir-126"=>"iso-8859-7",
"iso8859-7"=>"iso-8859-7",
"iso88597"=>"iso-8859-7",
"iso_8859-7"=>"iso-8859-7",
"iso_8859-7:1987"=>"iso-8859-7",
"sun_eu_greek"=>"iso-8859-7",
"csiso88598e"=>"iso-8859-8",
"csisolatinhebrew"=>"iso-8859-8",
"hebrew"=>"iso-8859-8",
"iso-8859-8"=>"iso-8859-8",
"iso-8859-8-e"=>"iso-8859-8",
"iso-ir-138"=>"iso-8859-8",
"iso8859-8"=>"iso-8859-8",
"iso88598"=>"iso-8859-8",
"iso_8859-8"=>"iso-8859-8",
"iso_8859-8:1988"=>"iso-8859-8",
"visual"=>"iso-8859-8",
"csisolatin6"=>"iso-8859-10",
"iso-8859-10"=>"iso-8859-10",
"iso-ir-157"=>"iso-8859-10",
"iso8859-10"=>"iso-8859-10",
"iso885910"=>"iso-8859-10",
"l6"=>"iso-8859-10",
"latin6"=>"iso-8859-10",
"iso-8859-13"=>"iso-8859-13",
"iso8859-13"=>"iso-8859-13",
"iso885913"=>"iso-8859-13",
"iso-8859-14"=>"iso-8859-14",
"iso8859-14"=>"iso-8859-14",
"iso885914"=>"iso-8859-14",
"csisolatin9"=>"iso-8859-15",
"iso-8859-15"=>"iso-8859-15",
"iso8859-15"=>"iso-8859-15",
"iso885915"=>"iso-8859-15",
"iso_8859-15"=>"iso-8859-15",
"l9"=>"iso-8859-15",
"iso-8859-16"=>"iso-8859-16",
"cskoi8r"=>"koi8-r",
"koi"=>"koi8-r",
"koi8"=>"koi8-r",
"koi8-r"=>"koi8-r",
"koi8_r"=>"koi8-r",
"koi8-u"=>"koi8-u",
"dos-874"=>"windows-874",
"iso-8859-11"=>"windows-874",
"iso8859-11"=>"windows-874",
"iso885911"=>"windows-874",
"tis-620"=>"windows-874",
"windows-874"=>"windows-874",
"cp1250"=>"windows-1250",
"windows-1250"=>"windows-1250",
"x-cp1250"=>"windows-1250",
"cp1251"=>"windows-1251",
"windows-1251"=>"windows-1251",
"x-cp1251"=>"windows-1251",
"ansi_x3.4-1968"=>"windows-1252",
"ascii"=>"windows-1252",
"cp1252"=>"windows-1252",
"cp819"=>"windows-1252",
"csisolatin1"=>"windows-1252",
"ibm819"=>"windows-1252",
"iso-8859-1"=>"windows-1252",
"iso-ir-100"=>"windows-1252",
"iso8859-1"=>"windows-1252",
"iso88591"=>"windows-1252",
"iso_8859-1"=>"windows-1252",
"iso_8859-1:1987"=>"windows-1252",
"l1"=>"windows-1252",
"latin1"=>"windows-1252",
"us-ascii"=>"windows-1252",
"windows-1252"=>"windows-1252",
"x-cp1252"=>"windows-1252",
"cp1253"=>"windows-1253",
"windows-1253"=>"windows-1253",
"x-cp1253"=>"windows-1253",
"cp1254"=>"windows-1254",
"csisolatin5"=>"windows-1254",
"iso-8859-9"=>"windows-1254",
"iso-ir-148"=>"windows-1254",
"iso8859-9"=>"windows-1254",
"iso88599"=>"windows-1254",
"iso_8859-9"=>"windows-1254",
"iso_8859-9:1989"=>"windows-1254",
"l5"=>"windows-1254",
"latin5"=>"windows-1254",
"windows-1254"=>"windows-1254",
"x-cp1254"=>"windows-1254",
"cp1255"=>"windows-1255",
"windows-1255"=>"windows-1255",
"x-cp1255"=>"windows-1255",
"cp1256"=>"windows-1256",
"windows-1256"=>"windows-1256",
"x-cp1256"=>"windows-1256",
"cp1257"=>"windows-1257",
"windows-1257"=>"windows-1257",
"x-cp1257"=>"windows-1257",
"cp1258"=>"windows-1258",
"windows-1258"=>"windows-1258",
"x-cp1258"=>"windows-1258",
"x-mac-cyrillic"=>"macCyrillic",
"x-mac-ukrainian"=>"macCyrillic",
"chinese"=>"gbk",
"csgb2312"=>"gbk",
"csiso58gb231280"=>"gbk",
"gb2312"=>"gbk",
"gb_2312"=>"gbk",
"gb_2312-80"=>"gbk",
"gbk"=>"gbk",
"iso-ir-58"=>"gbk",
"x-gbk"=>"gbk",
"gb18030"=>"gb18030",
"big5"=>"big5",
"big5-hkscs"=>"big5",
"cn-big5"=>"big5",
"csbig5"=>"big5",
"x-x-big5"=>"big5",
"cseucpkdfmtjapanese"=>"cp51932",
"euc-jp"=>"cp51932",
"x-euc-jp"=>"cp51932",
"csiso2022jp"=>"cp50221",
"iso-2022-jp"=>"cp50221",
"csshiftjis"=>"Windows-31J",
"ms_kanji"=>"Windows-31J",
"shift-jis"=>"Windows-31J",
"shift_jis"=>"Windows-31J",
"sjis"=>"Windows-31J",
"windows-31j"=>"Windows-31J",
"x-sjis"=>"Windows-31J",
"cseuckr"=>"euc-kr",
"csksc56011987"=>"euc-kr",
"euc-kr"=>"euc-kr",
"iso-ir-149"=>"euc-kr",
"korean"=>"euc-kr",
"ks_c_5601-1987"=>"euc-kr",
"ks_c_5601-1989"=>"euc-kr",
"ksc5601"=>"euc-kr",
"ksc_5601"=>"euc-kr",
"windows-949"=>"euc-kr",
"utf-16be"=>"utf-16be",
"utf-16"=>"utf-16le",
"utf-16le"=>"utf-16le"
} # :nodoc:
# :nodoc:
# return encoding or nil
# http://encoding.spec.whatwg.org/#concept-encoding-get
def self.get_encoding(label)
Encoding.find(WEB_ENCODINGS_[label.to_str.strip.downcase]) rescue nil
end
end # module URI
module Kernel

View file

@ -56,10 +56,32 @@ class TestCommon < Test::Unit::TestCase
URI.encode_www_form_component("\x00 !\"\#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~"))
assert_equal("%95A", URI.encode_www_form_component(
"\x95\x41".force_encoding(Encoding::Shift_JIS)))
assert_equal("%E3%81%82", URI.encode_www_form_component(
assert_equal("0B", URI.encode_www_form_component(
"\x30\x42".force_encoding(Encoding::UTF_16BE)))
assert_equal("%1B%24B%24%22%1B%28B", URI.encode_www_form_component(
"\e$B$\"\e(B".force_encoding(Encoding::ISO_2022_JP)))
assert_equal("%E3%81%82", URI.encode_www_form_component(
"\u3042", Encoding::ASCII_8BIT))
assert_equal("%82%A0", URI.encode_www_form_component(
"\u3042", Encoding::Windows_31J))
assert_equal("%E3%81%82", URI.encode_www_form_component(
"\u3042", Encoding::UTF_8))
assert_equal("%82%A0", URI.encode_www_form_component(
"\u3042".encode("sjis"), Encoding::ASCII_8BIT))
assert_equal("%A4%A2", URI.encode_www_form_component(
"\u3042".encode("sjis"), Encoding::EUC_JP))
assert_equal("%E3%81%82", URI.encode_www_form_component(
"\u3042".encode("sjis"), Encoding::UTF_8))
assert_equal("B0", URI.encode_www_form_component(
"\u3042".encode("sjis"), Encoding::UTF_16LE))
# invalid
assert_equal("%EF%BF%BD%EF%BF%BD", URI.encode_www_form_component(
"\xE3\x81\xFF", "utf-8"))
assert_equal("%E6%9F%8A%EF%BF%BD%EF%BF%BD", URI.encode_www_form_component(
"\x95\x41\xff\xff".force_encoding(Encoding::Shift_JIS), "utf-8"))
end
def test_decode_www_form_component
@ -82,6 +104,8 @@ class TestCommon < Test::Unit::TestCase
assert_equal(expected, URI.encode_www_form(a: 1, :"\u3042" => "\u6F22"))
assert_equal(expected, URI.encode_www_form([["a", "1"], ["\u3042", "\u6F22"]]))
assert_equal(expected, URI.encode_www_form([[:a, 1], [:"\u3042", "\u6F22"]]))
assert_equal("a=1&%82%A0=%8A%BF",
URI.encode_www_form({"a" => "1", "\u3042" => "\u6F22"}, "sjis"))
assert_equal('+a+=+1+', URI.encode_www_form([[' a ', ' 1 ']]))
assert_equal('text=x%0Ay', URI.encode_www_form([['text', "x\u000Ay"]]))
@ -106,18 +130,32 @@ class TestCommon < Test::Unit::TestCase
def test_decode_www_form
assert_equal([%w[a 1], %w[a 2]], URI.decode_www_form("a=1&a=2"))
assert_equal([%w[a 1;a=2]], URI.decode_www_form("a=1;a=2"))
assert_equal([%w[a 1], ['', ''], %w[a 2]], URI.decode_www_form("a=1&&a=2"))
assert_raise(ArgumentError){URI.decode_www_form("\u3042")}
assert_equal([%w[a 1], ["\u3042", "\u6F22"]],
URI.decode_www_form("a=1;%E3%81%82=%E6%BC%A2"))
URI.decode_www_form("a=1&%E3%81%82=%E6%BC%A2"))
assert_equal([%w[?a 1], %w[a 2]], URI.decode_www_form("?a=1&a=2"))
assert_equal([], URI.decode_www_form(""))
assert_raise(ArgumentError){URI.decode_www_form("%=1")}
assert_raise(ArgumentError){URI.decode_www_form("a=%")}
assert_raise(ArgumentError){URI.decode_www_form("a=1&%=2")}
assert_raise(ArgumentError){URI.decode_www_form("a=1&b=%")}
assert_raise(ArgumentError){URI.decode_www_form("a&b")}
assert_equal([%w[% 1]], URI.decode_www_form("%=1"))
assert_equal([%w[a %]], URI.decode_www_form("a=%"))
assert_equal([%w[a 1], %w[% 2]], URI.decode_www_form("a=1&%=2"))
assert_equal([%w[a 1], %w[b %]], URI.decode_www_form("a=1&b=%"))
assert_equal([['a', ''], ['b', '']], URI.decode_www_form("a&b"))
bug4098 = '[ruby-core:33464]'
assert_raise(ArgumentError, bug4098){URI.decode_www_form("a=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&b")}
assert_equal([['a', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'], ['b', '']], URI.decode_www_form("a=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&b"), bug4098)
assert_raise(ArgumentError){ URI.decode_www_form("a=1&%82%A0=%8A%BF", "x-sjis") }
assert_equal([["a", "1"], [s("\x82\xA0"), s("\x8a\xBF")]],
URI.decode_www_form("a=1&%82%A0=%8A%BF", "sjis"))
assert_equal([["a", "1"], [s("\x82\xA0"), s("\x8a\xBF")], %w[_charset_ sjis], [s("\x82\xA1"), s("\x8a\xC0")]],
URI.decode_www_form("a=1&%82%A0=%8A%BF&_charset_=sjis&%82%A1=%8A%C0", use__charset_: true))
assert_equal([["", "isindex"], ["a", "1"]],
URI.decode_www_form("isindex&a=1", isindex: true))
end
private
def s(str) str.force_encoding(Encoding::Windows_31J); end
end