diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md
index 3db7d2efd9..fddac3d429 100644
--- a/activesupport/CHANGELOG.md
+++ b/activesupport/CHANGELOG.md
@@ -1,3 +1,7 @@
+* Remove deprecated `ActiveSupport::Multibyte::Chars#consumes?` and `ActiveSupport::Multibyte::Chars#normalize`.
+
+ *Rafael Mendonça França*
+
* Remove deprecated file `active_support/core_ext/range/include_range`.
*Rafael Mendonça França*
diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb
index f58baa0a67..d58ab1ba05 100644
--- a/activesupport/lib/active_support/multibyte/chars.rb
+++ b/activesupport/lib/active_support/multibyte/chars.rb
@@ -74,17 +74,6 @@ module ActiveSupport #:nodoc:
@wrapped_string.respond_to?(method, include_private)
end
- # Returns +true+ when the proxy class can handle the string. Returns
- # +false+ otherwise.
- def self.consumes?(string)
- ActiveSupport::Deprecation.warn(<<-MSG.squish)
- ActiveSupport::Multibyte::Chars.consumes? is deprecated and will be
- removed from Rails 6.1. Use string.is_utf8? instead.
- MSG
-
- string.encoding == Encoding::UTF_8
- end
-
# Works just like String#split, with the exception that the items
# in the resulting list are Chars instances instead of String. This makes
# chaining methods easier.
@@ -135,34 +124,6 @@ module ActiveSupport #:nodoc:
end
alias_method :titlecase, :titleize
- # Returns the KC normalization of the string by default. NFKC is
- # considered the best normalization form for passing strings to databases
- # and validations.
- #
- # * form - The form you want to normalize in. Should be one of the following:
- # :c, :kc, :d, or :kd. Default is
- # ActiveSupport::Multibyte::Unicode.default_normalization_form
- def normalize(form = nil)
- form ||= Unicode.default_normalization_form
-
- # See https://www.unicode.org/reports/tr15, Table 1
- if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form]
- ActiveSupport::Deprecation.warn(<<-MSG.squish)
- ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
- removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead.
- MSG
-
- unicode_normalize(alias_form)
- else
- ActiveSupport::Deprecation.warn(<<-MSG.squish)
- ActiveSupport::Multibyte::Chars#normalize is deprecated and will be
- removed from Rails 6.1. Use #unicode_normalize instead.
- MSG
-
- raise ArgumentError, "#{form} is not a valid normalization variant", caller
- end
- end
-
# Performs canonical decomposition on all the characters.
#
# 'é'.length # => 2
diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb
index 1200c79b68..35560ede23 100644
--- a/activesupport/test/multibyte_chars_test.rb
+++ b/activesupport/test/multibyte_chars_test.rb
@@ -72,18 +72,6 @@ class MultibyteCharsTest < ActiveSupport::TestCase
assert_equal "abb", mb_a << mb_b
end
- def test_consumes_utf8_strings
- ActiveSupport::Deprecation.silence do
- assert @proxy_class.consumes?(UNICODE_STRING)
- assert @proxy_class.consumes?(ASCII_STRING)
- assert_not @proxy_class.consumes?(BYTE_STRING)
- end
- end
-
- def test_consumes_is_deprecated
- assert_deprecated { @proxy_class.consumes?(UNICODE_STRING) }
- end
-
def test_concatenation_should_return_a_proxy_class_instance
assert_equal ActiveSupport::Multibyte.proxy_class, ("a".mb_chars + "b").class
assert_equal ActiveSupport::Multibyte.proxy_class, ((+"a").mb_chars << "b").class
@@ -171,9 +159,6 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase
assert chars("").upcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars("").downcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars("").capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
- ActiveSupport::Deprecation.silence do
- assert chars("").normalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
- end
assert chars("").decompose.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars("").compose.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars("").tidy_bytes.kind_of?(ActiveSupport::Multibyte.proxy_class)
@@ -397,12 +382,6 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase
def test_reverse_should_work_with_normalized_strings
str = "bös"
reversed_str = "söb"
- ActiveSupport::Deprecation.silence do
- assert_equal chars(reversed_str).normalize(:kc), chars(str).normalize(:kc).reverse
- assert_equal chars(reversed_str).normalize(:c), chars(str).normalize(:c).reverse
- assert_equal chars(reversed_str).normalize(:d), chars(str).normalize(:d).reverse
- assert_equal chars(reversed_str).normalize(:kd), chars(str).normalize(:kd).reverse
- end
assert_equal chars(reversed_str).decompose, chars(str).decompose.reverse
assert_equal chars(reversed_str).compose, chars(str).compose.reverse
end
@@ -581,57 +560,13 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
end
end
- def test_composition_exclusion_is_set_up_properly
- # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
- qa = [0x915, 0x93c].pack("U*")
- ActiveSupport::Deprecation.silence do
- assert_equal qa, chars(qa).normalize(:c)
- end
- end
-
- # Test for the Public Review Issue #29, bad explanation of composition might lead to a
- # bad implementation: http://www.unicode.org/review/pr-29.html
- def test_normalization_C_pri_29
- [
- [0x0B47, 0x0300, 0x0B3E],
- [0x1100, 0x0300, 0x1161]
- ].map { |c| c.pack("U*") }.each do |c|
- ActiveSupport::Deprecation.silence do
- assert_equal_codepoints c, chars(c).normalize(:c)
- end
- end
- end
-
def test_normalization_shouldnt_strip_null_bytes
null_byte_str = "Test\0test"
- ActiveSupport::Deprecation.silence do
- assert_equal null_byte_str, chars(null_byte_str).normalize(:kc)
- assert_equal null_byte_str, chars(null_byte_str).normalize(:c)
- assert_equal null_byte_str, chars(null_byte_str).normalize(:d)
- assert_equal null_byte_str, chars(null_byte_str).normalize(:kd)
- end
assert_equal null_byte_str, chars(null_byte_str).decompose
assert_equal null_byte_str, chars(null_byte_str).compose
end
- def test_simple_normalization
- comp_str = [
- 44, # LATIN CAPITAL LETTER D
- 307, # COMBINING DOT ABOVE
- 328, # COMBINING OGONEK
- 323 # COMBINING DOT BELOW
- ].pack("U*")
-
- ActiveSupport::Deprecation.silence do
- assert_equal_codepoints "", chars("").normalize
- assert_equal_codepoints [44, 105, 106, 328, 323].pack("U*"), chars(comp_str).normalize(:kc).to_s
- assert_equal_codepoints [44, 307, 328, 323].pack("U*"), chars(comp_str).normalize(:c).to_s
- assert_equal_codepoints [44, 307, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:d).to_s
- assert_equal_codepoints [44, 105, 106, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:kd).to_s
- end
- end
-
def test_should_compute_grapheme_length
[
["", 0],
@@ -757,21 +692,6 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase
end
end
- def test_chars_normalize_deprecation
- # String#unicode_normalize default form is `:nfc`, and
- # different than Multibyte::Unicode default, `:nkfc`.
- # Deprecation should suggest the right form if no params
- # are given and default is used.
- assert_deprecated(/unicode_normalize\(:nfkc\)/) do
- "".mb_chars.normalize
- end
-
- assert_deprecated(/unicode_normalize\(:nfc\)/) { "".mb_chars.normalize(:c) }
- assert_deprecated(/unicode_normalize\(:nfd\)/) { "".mb_chars.normalize(:d) }
- assert_deprecated(/unicode_normalize\(:nfkc\)/) { "".mb_chars.normalize(:kc) }
- assert_deprecated(/unicode_normalize\(:nfkd\)/) { "".mb_chars.normalize(:kd) }
- end
-
def test_unicode_deprecations
assert_deprecated { ActiveSupport::Multibyte::Unicode.downcase("") }
assert_deprecated { ActiveSupport::Multibyte::Unicode.upcase("") }
diff --git a/activesupport/test/multibyte_conformance_test.rb b/activesupport/test/multibyte_conformance_test.rb
deleted file mode 100644
index 95dab84194..0000000000
--- a/activesupport/test/multibyte_conformance_test.rb
+++ /dev/null
@@ -1,113 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "abstract_unit"
-require_relative "multibyte_test_helpers"
-
-class MultibyteConformanceTest < ActiveSupport::TestCase
- include MultibyteTestHelpers
-
- UNIDATA_FILE = "/NormalizationTest.txt"
- RUN_P = begin
- Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
- rescue
- end
-
- def setup
- @proxy = ActiveSupport::Multibyte::Chars
- skip "Unable to download test data" unless RUN_P
- end
-
- def test_normalizations_C
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do |*cols|
- col1, col2, col3, col4, col5, comment = *cols
-
- # CONFORMANCE:
- # 1. The following invariants must be true for all conformant implementations
- #
- # NFC
- # c2 == NFC(c1) == NFC(c2) == NFC(c3)
- assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
- assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
- assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
- #
- # c4 == NFC(c4) == NFC(c5)
- assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_D
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do |*cols|
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFD
- # c3 == NFD(c1) == NFD(c2) == NFD(c3)
- assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
- assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
- assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
- # c5 == NFD(c4) == NFD(c5)
- assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_KC
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do | *cols |
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFKC
- # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
- assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_KD
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do | *cols |
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFKD
- # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
- assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
- end
- end
- end
-
- private
- def each_line_of_norm_tests(&block)
- File.open(File.join(CACHE_DIR, UNIDATA_FILE), "r") do | f |
- until f.eof?
- line = f.gets.chomp!
- next if line.empty? || line.start_with?("#")
-
- cols, comment = line.split("#")
- cols = cols.split(";").map(&:strip).reject(&:empty?)
- next unless cols.length == 5
-
- # codepoints are in hex in the test suite, pack wants them as integers
- cols.map! { |c| c.split.map { |codepoint| codepoint.to_i(16) }.pack("U*") }
- cols << comment
-
- yield(*cols)
- end
- end
- end
-
- def inspect_codepoints(str)
- str.to_s.unpack("U*").map { |cp| cp.to_s(16) }.join(" ")
- end
-end
diff --git a/activesupport/test/multibyte_normalization_conformance_test.rb b/activesupport/test/multibyte_normalization_conformance_test.rb
deleted file mode 100644
index 581a1a32d1..0000000000
--- a/activesupport/test/multibyte_normalization_conformance_test.rb
+++ /dev/null
@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "abstract_unit"
-require_relative "multibyte_test_helpers"
-
-class MultibyteNormalizationConformanceTest < ActiveSupport::TestCase
- include MultibyteTestHelpers
-
- UNIDATA_FILE = "/NormalizationTest.txt"
- RUN_P = begin
- Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE)
- rescue
- end
-
- def setup
- @proxy = ActiveSupport::Multibyte::Chars
- skip "Unable to download test data" unless RUN_P
- end
-
- def test_normalizations_C
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do |*cols|
- col1, col2, col3, col4, col5, comment = *cols
-
- # CONFORMANCE:
- # 1. The following invariants must be true for all conformant implementations
- #
- # NFC
- # c2 == NFC(c1) == NFC(c2) == NFC(c3)
- assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}"
- assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}"
- assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}"
- #
- # c4 == NFC(c4) == NFC(c5)
- assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_D
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do |*cols|
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFD
- # c3 == NFD(c1) == NFD(c2) == NFD(c3)
- assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}"
- assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}"
- assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}"
- # c5 == NFD(c4) == NFD(c5)
- assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_KC
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do | *cols |
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFKC
- # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
- assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}"
- assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}"
- end
- end
- end
-
- def test_normalizations_KD
- ActiveSupport::Deprecation.silence do
- each_line_of_norm_tests do | *cols |
- col1, col2, col3, col4, col5, comment = *cols
- #
- # NFKD
- # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
- assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}"
- assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}"
- end
- end
- end
-
- private
- def each_line_of_norm_tests(&block)
- lines = 0
- max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile
- File.open(File.join(CACHE_DIR, UNIDATA_FILE), "r") do | f |
- until f.eof? || (max_test_lines > 38 && lines > max_test_lines)
- lines += 1
- line = f.gets.chomp!
- next if line.empty? || line.start_with?("#")
-
- cols, comment = line.split("#")
- cols = cols.split(";").map { |e| e.strip }.reject { |e| e.empty? }
- next unless cols.length == 5
-
- # codepoints are in hex in the test suite, pack wants them as integers
- cols.map! { |c| c.split.map { |codepoint| codepoint.to_i(16) }.pack("U*") }
- cols << comment
-
- yield(*cols)
- end
- end
- end
-
- def inspect_codepoints(str)
- str.to_s.unpack("U*").map { |cp| cp.to_s(16) }.join(" ")
- end
-end
diff --git a/guides/source/6_1_release_notes.md b/guides/source/6_1_release_notes.md
index 7ac80238f3..6f6e073f39 100644
--- a/guides/source/6_1_release_notes.md
+++ b/guides/source/6_1_release_notes.md
@@ -304,6 +304,8 @@ Please refer to the [Changelog][active-support] for detailed changes.
* Remove deprecated file `active_support/core_ext/range/include_range`.
+* Remove deprecated `ActiveSupport::Multibyte::Chars#consumes?` and `ActiveSupport::Multibyte::Chars#normalize`.
+
### Deprecations
### Notable changes