Deprecate Unicode#normalize and Chars#normalize (#34202)
This commit is contained in:
parent
619b2ae6a4
commit
ee95bed3e6
|
@ -1,3 +1,8 @@
|
|||
* Deprecate `ActiveSupport::Multibyte::Unicode#normalize` and `ActiveSuppport::Multibyte::Chars#normalize`
|
||||
in favor of `String#unicode_normalize`
|
||||
|
||||
*Francesco Rodríguez*
|
||||
|
||||
* Deprecate `ActiveSupport::Multibyte::Unicode#downcase/upcase/swapcase` in favor of
|
||||
`String#downcase/upcase/swapcase`.
|
||||
|
||||
|
|
|
@ -62,9 +62,9 @@ module ActiveSupport
|
|||
raise ArgumentError, "Can only transliterate strings. Received #{string.class.name}" unless string.is_a?(String)
|
||||
|
||||
I18n.transliterate(
|
||||
ActiveSupport::Multibyte::Unicode.normalize(
|
||||
ActiveSupport::Multibyte::Unicode.tidy_bytes(string), :c),
|
||||
replacement: replacement)
|
||||
ActiveSupport::Multibyte::Unicode.tidy_bytes(string).unicode_normalize(:nfc),
|
||||
replacement: replacement
|
||||
)
|
||||
end
|
||||
|
||||
# Replaces special characters in a string so that it may be used as part of
|
||||
|
|
|
@ -17,7 +17,7 @@ module ActiveSupport #:nodoc:
|
|||
# through the +mb_chars+ method. Methods which would normally return a
|
||||
# String object now return a Chars object so methods can be chained.
|
||||
#
|
||||
# 'The Perfect String '.mb_chars.downcase.strip.normalize
|
||||
# 'The Perfect String '.mb_chars.downcase.strip
|
||||
# # => #<ActiveSupport::Multibyte::Chars:0x007fdc434ccc10 @wrapped_string="the perfect string">
|
||||
#
|
||||
# Chars objects are perfectly interchangeable with String objects as long as
|
||||
|
@ -137,7 +137,24 @@ module ActiveSupport #:nodoc:
|
|||
# <tt>:c</tt>, <tt>:kc</tt>, <tt>:d</tt>, or <tt>:kd</tt>. Default is
|
||||
# ActiveSupport::Multibyte::Unicode.default_normalization_form
|
||||
def normalize(form = nil)
|
||||
chars(Unicode.normalize(@wrapped_string, form))
|
||||
form ||= Unicode.default_normalization_form
|
||||
|
||||
# See https://www.unicode.org/reports/tr15, Table 1
|
||||
if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form]
|
||||
ActiveSupport::Deprecation.warn(<<-MSG.squish)
|
||||
ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
|
||||
removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead.
|
||||
MSG
|
||||
|
||||
send(:unicode_normalize, alias_form)
|
||||
else
|
||||
ActiveSupport::Deprecation.warn(<<-MSG.squish)
|
||||
ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
|
||||
removed from Rails 6.1. Use #unicode_normalize instead.
|
||||
MSG
|
||||
|
||||
raise ArgumentError, "#{form} is not a valid normalization variant", caller
|
||||
end
|
||||
end
|
||||
|
||||
# Performs canonical decomposition on all the characters.
|
||||
|
|
|
@ -6,10 +6,17 @@ module ActiveSupport
|
|||
extend self
|
||||
|
||||
# A list of all available normalization forms.
|
||||
# See http://www.unicode.org/reports/tr15/tr15-29.html for more
|
||||
# See https://www.unicode.org/reports/tr15/tr15-29.html for more
|
||||
# information about normalization.
|
||||
NORMALIZATION_FORMS = [:c, :kc, :d, :kd]
|
||||
|
||||
NORMALIZATION_FORM_ALIASES = { # :nodoc:
|
||||
c: :nfc,
|
||||
d: :nfd,
|
||||
kc: :nfkc,
|
||||
kd: :nfkd
|
||||
}
|
||||
|
||||
# The Unicode version that is supported by the implementation
|
||||
UNICODE_VERSION = RbConfig::CONFIG["UNICODE_VERSION"]
|
||||
|
||||
|
@ -100,17 +107,21 @@ module ActiveSupport
|
|||
# Default is ActiveSupport::Multibyte::Unicode.default_normalization_form.
|
||||
def normalize(string, form = nil)
|
||||
form ||= @default_normalization_form
|
||||
# See http://www.unicode.org/reports/tr15, Table 1
|
||||
case form
|
||||
when :d
|
||||
string.unicode_normalize(:nfd)
|
||||
when :c
|
||||
string.unicode_normalize(:nfc)
|
||||
when :kd
|
||||
string.unicode_normalize(:nfkd)
|
||||
when :kc
|
||||
string.unicode_normalize(:nfkc)
|
||||
|
||||
# See https://www.unicode.org/reports/tr15, Table 1
|
||||
if alias_form = NORMALIZATION_FORM_ALIASES[form]
|
||||
ActiveSupport::Deprecation.warn(<<-MSG.squish)
|
||||
ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
|
||||
removed from Rails 6.1. Use String#unicode_normalize(:#{alias_form}) instead.
|
||||
MSG
|
||||
|
||||
string.unicode_normalize(alias_form)
|
||||
else
|
||||
ActiveSupport::Deprecation.warn(<<-MSG.squish)
|
||||
ActiveSupport::Multibyte::Unicode#normalize is deprecated and will be
|
||||
removed from Rails 6.1. Use String#unicode_normalize instead.
|
||||
MSG
|
||||
|
||||
raise ArgumentError, "#{form} is not a valid normalization variant", caller
|
||||
end
|
||||
end
|
||||
|
|
|
@ -165,7 +165,9 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase
|
|||
assert chars("").upcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
assert chars("").downcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
assert chars("").capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
assert chars("").normalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert chars("").normalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
end
|
||||
assert chars("").decompose.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
assert chars("").compose.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
assert chars("").tidy_bytes.kind_of?(ActiveSupport::Multibyte.proxy_class)
|
||||
|
@ -383,10 +385,12 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase
|
|||
def test_reverse_should_work_with_normalized_strings
|
||||
str = "bös"
|
||||
reversed_str = "söb"
|
||||
assert_equal chars(reversed_str).normalize(:kc), chars(str).normalize(:kc).reverse
|
||||
assert_equal chars(reversed_str).normalize(:c), chars(str).normalize(:c).reverse
|
||||
assert_equal chars(reversed_str).normalize(:d), chars(str).normalize(:d).reverse
|
||||
assert_equal chars(reversed_str).normalize(:kd), chars(str).normalize(:kd).reverse
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert_equal chars(reversed_str).normalize(:kc), chars(str).normalize(:kc).reverse
|
||||
assert_equal chars(reversed_str).normalize(:c), chars(str).normalize(:c).reverse
|
||||
assert_equal chars(reversed_str).normalize(:d), chars(str).normalize(:d).reverse
|
||||
assert_equal chars(reversed_str).normalize(:kd), chars(str).normalize(:kd).reverse
|
||||
end
|
||||
assert_equal chars(reversed_str).decompose, chars(str).decompose.reverse
|
||||
assert_equal chars(reversed_str).compose, chars(str).compose.reverse
|
||||
end
|
||||
|
@ -568,7 +572,9 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
|
|||
def test_composition_exclusion_is_set_up_properly
|
||||
# Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
|
||||
qa = [0x915, 0x93c].pack("U*")
|
||||
assert_equal qa, chars(qa).normalize(:c)
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert_equal qa, chars(qa).normalize(:c)
|
||||
end
|
||||
end
|
||||
|
||||
# Test for the Public Review Issue #29, bad explanation of composition might lead to a
|
||||
|
@ -578,17 +584,21 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
|
|||
[0x0B47, 0x0300, 0x0B3E],
|
||||
[0x1100, 0x0300, 0x1161]
|
||||
].map { |c| c.pack("U*") }.each do |c|
|
||||
assert_equal_codepoints c, chars(c).normalize(:c)
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert_equal_codepoints c, chars(c).normalize(:c)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalization_shouldnt_strip_null_bytes
|
||||
null_byte_str = "Test\0test"
|
||||
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
|
||||
assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
|
||||
end
|
||||
assert_equal null_byte_str, chars(null_byte_str).decompose
|
||||
assert_equal null_byte_str, chars(null_byte_str).compose
|
||||
end
|
||||
|
@ -601,11 +611,13 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
|
|||
323 # COMBINING DOT BELOW
|
||||
].pack("U*")
|
||||
|
||||
assert_equal_codepoints "", chars("").normalize
|
||||
assert_equal_codepoints [44, 105, 106, 328, 323].pack("U*"), chars(comp_str).normalize(:kc).to_s
|
||||
assert_equal_codepoints [44, 307, 328, 323].pack("U*"), chars(comp_str).normalize(:c).to_s
|
||||
assert_equal_codepoints [44, 307, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:d).to_s
|
||||
assert_equal_codepoints [44, 105, 106, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:kd).to_s
|
||||
ActiveSupport::Deprecation.silence do
|
||||
assert_equal_codepoints "", chars("").normalize
|
||||
assert_equal_codepoints [44, 105, 106, 328, 323].pack("U*"), chars(comp_str).normalize(:kc).to_s
|
||||
assert_equal_codepoints [44, 307, 328, 323].pack("U*"), chars(comp_str).normalize(:c).to_s
|
||||
assert_equal_codepoints [44, 307, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:d).to_s
|
||||
assert_equal_codepoints [44, 105, 106, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:kd).to_s
|
||||
end
|
||||
end
|
||||
|
||||
def test_should_compute_grapheme_length
|
||||
|
@ -719,6 +731,35 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
|
|||
assert_equal BYTE_STRING.dup.mb_chars.class, ActiveSupport::Multibyte::Chars
|
||||
end
|
||||
|
||||
def test_unicode_normalize_deprecation
|
||||
# String#unicode_normalize default form is `:nfc`, and
|
||||
# different than Multibyte::Unicode default, `:nkfc`.
|
||||
# Deprecation should suggest the right form if no params
|
||||
# are given and default is used.
|
||||
assert_deprecated(/unicode_normalize\(:nfkc\)/) do
|
||||
ActiveSupport::Multibyte::Unicode.normalize("")
|
||||
end
|
||||
|
||||
assert_deprecated(/unicode_normalize\(:nfd\)/) do
|
||||
ActiveSupport::Multibyte::Unicode.normalize("", :d)
|
||||
end
|
||||
end
|
||||
|
||||
def test_chars_normalize_deprecation
|
||||
# String#unicode_normalize default form is `:nfc`, and
|
||||
# different than Multibyte::Unicode default, `:nkfc`.
|
||||
# Deprecation should suggest the right form if no params
|
||||
# are given and default is used.
|
||||
assert_deprecated(/unicode_normalize\(:nfkc\)/) do
|
||||
"".mb_chars.normalize
|
||||
end
|
||||
|
||||
assert_deprecated(/unicode_normalize\(:nfc\)/) { "".mb_chars.normalize(:c) }
|
||||
assert_deprecated(/unicode_normalize\(:nfd\)/) { "".mb_chars.normalize(:d) }
|
||||
assert_deprecated(/unicode_normalize\(:nfkc\)/) { "".mb_chars.normalize(:kc) }
|
||||
assert_deprecated(/unicode_normalize\(:nfkd\)/) { "".mb_chars.normalize(:kd) }
|
||||
end
|
||||
|
||||
def test_unicode_deprecations
|
||||
assert_deprecated { ActiveSupport::Multibyte::Unicode.downcase("") }
|
||||
assert_deprecated { ActiveSupport::Multibyte::Unicode.upcase("") }
|
||||
|
|
|
@ -18,64 +18,72 @@ class MultibyteConformanceTest < ActiveSupport::TestCase
|
|||
end
|
||||
|
||||
def test_normalizations_C
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
|
||||
# CONFORMANCE:
|
||||
# 1. The following invariants must be true for all conformant implementations
|
||||
#
|
||||
# NFC
|
||||
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
|
||||
#
|
||||
# c4 == NFC(c4) == NFC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
|
||||
# CONFORMANCE:
|
||||
# 1. The following invariants must be true for all conformant implementations
|
||||
#
|
||||
# NFC
|
||||
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
|
||||
#
|
||||
# c4 == NFC(c4) == NFC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_D
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFD
|
||||
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
|
||||
# c5 == NFD(c4) == NFD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFD
|
||||
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
|
||||
# c5 == NFD(c4) == NFD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_KC
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKC
|
||||
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKC
|
||||
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_KD
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKD
|
||||
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKD
|
||||
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -18,64 +18,72 @@ class MultibyteNormalizationConformanceTest < ActiveSupport::TestCase
|
|||
end
|
||||
|
||||
def test_normalizations_C
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
|
||||
# CONFORMANCE:
|
||||
# 1. The following invariants must be true for all conformant implementations
|
||||
#
|
||||
# NFC
|
||||
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
|
||||
#
|
||||
# c4 == NFC(c4) == NFC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
|
||||
# CONFORMANCE:
|
||||
# 1. The following invariants must be true for all conformant implementations
|
||||
#
|
||||
# NFC
|
||||
# c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
|
||||
assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
|
||||
#
|
||||
# c4 == NFC(c4) == NFC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_D
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFD
|
||||
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
|
||||
# c5 == NFD(c4) == NFD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do |*cols|
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFD
|
||||
# c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
|
||||
assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
|
||||
# c5 == NFD(c4) == NFD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_KC
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKC
|
||||
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKC
|
||||
# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
|
||||
assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_normalizations_KD
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKD
|
||||
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
|
||||
ActiveSupport::Deprecation.silence do
|
||||
each_line_of_norm_tests do | *cols |
|
||||
col1, col2, col3, col4, col5, comment = *cols
|
||||
#
|
||||
# NFKD
|
||||
# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
|
||||
assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
Loading…
Reference in New Issue