kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

add test_ill_formed_utf_8_replace to test recommended number of \uFFFD

Browse source 
Add test method test_ill_formed_utf_8_replace to test/ruby/test_transcode.rb
to check for the recommended number of \uFFFD replacement characters.
This is the first part, using ill-formed prefixes, with suffixes up to
the length of the original UTF-8 structure (including overlongs and
the full 31-bit space.)

For more details, see Unicode 9.0.0, Section 3.9, Best Practices for Using U+FFFD.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59026 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
duerst 2017-06-06 10:34:10 +00:00
parent 7d041e87c6
commit 267323cad5
1 changed files with 74 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
test/ruby/test_transcode.rb
View file

@ -998,6 +998,80 @@ class TestTranscode < Test::Unit::TestCase
check_both_ways("\u00A0", "\xFF", 'CP855') # non-breaking space
end
def test_ill_formed_utf_8_replace
fffd1 = "\uFFFD".encode 'UTF-16BE'
fffd2 = "\uFFFD\uFFFD".encode 'UTF-16BE'
fffd3 = "\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
fffd4 = "\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
fffd5 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
fffd6 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
assert_equal fffd2, "\xC0\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xC0\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xC1\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xC1\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xE0\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xE0\x9F".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xE0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xE0\x9F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xED\xA0".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xED\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xED\xA0\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xED\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF0\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF0\x8F".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF0\x8F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF0\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF0\x8F\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF4\x90".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF4\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF4\x90\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF4\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF4\x90\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF4\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF5\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF7\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF5\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF7\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF5\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF7\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xF8".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xFB".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xF8\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xFB\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xF8\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xFB\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xF8\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xFB\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xF8\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xFB\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xFC".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xFD".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xFC\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xFD\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xFC\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xFD\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xFC\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xFD\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xFC\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xFD\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd6, "\xFC\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd6, "\xFD\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xFE".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd1, "\xFF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xFE\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd2, "\xFF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xFE\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd3, "\xFF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xFE\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd4, "\xFF\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xFE\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd5, "\xFF\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd6, "\xFE\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
assert_equal fffd6, "\xFF\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
end
def check_utf_16_both_ways(utf8, raw)
copy = raw.dup
0.step(copy.length-1, 2) { |i| copy[i+1], copy[i] = copy[i], copy[i+1] }