From 8f14d5ad4b3aa3fbdd8051276caec1b92f937e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mendon=C3=A7a=20Fran=C3=A7a?= Date: Wed, 28 Oct 2020 22:25:13 +0000 Subject: [PATCH] Remove deprecated `ActiveSupport::Multibyte::Chars#consumes?` and `ActiveSupport::Multibyte::Chars#normalize` --- activesupport/CHANGELOG.md | 4 + .../lib/active_support/multibyte/chars.rb | 39 ------ activesupport/test/multibyte_chars_test.rb | 80 ------------ .../test/multibyte_conformance_test.rb | 113 ----------------- ...ultibyte_normalization_conformance_test.rb | 116 ------------------ guides/source/6_1_release_notes.md | 2 + 6 files changed, 6 insertions(+), 348 deletions(-) delete mode 100644 activesupport/test/multibyte_conformance_test.rb delete mode 100644 activesupport/test/multibyte_normalization_conformance_test.rb diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md index 3db7d2efd9..fddac3d429 100644 --- a/activesupport/CHANGELOG.md +++ b/activesupport/CHANGELOG.md @@ -1,3 +1,7 @@ +* Remove deprecated `ActiveSupport::Multibyte::Chars#consumes?` and `ActiveSupport::Multibyte::Chars#normalize`. + + *Rafael Mendonça França* + * Remove deprecated file `active_support/core_ext/range/include_range`. *Rafael Mendonça França* diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index f58baa0a67..d58ab1ba05 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -74,17 +74,6 @@ module ActiveSupport #:nodoc: @wrapped_string.respond_to?(method, include_private) end - # Returns +true+ when the proxy class can handle the string. Returns - # +false+ otherwise. - def self.consumes?(string) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - ActiveSupport::Multibyte::Chars.consumes? is deprecated and will be - removed from Rails 6.1. Use string.is_utf8? instead. - MSG - - string.encoding == Encoding::UTF_8 - end - # Works just like String#split, with the exception that the items # in the resulting list are Chars instances instead of String. This makes # chaining methods easier. @@ -135,34 +124,6 @@ module ActiveSupport #:nodoc: end alias_method :titlecase, :titleize - # Returns the KC normalization of the string by default. NFKC is - # considered the best normalization form for passing strings to databases - # and validations. - # - # * form - The form you want to normalize in. Should be one of the following: - # :c, :kc, :d, or :kd. Default is - # ActiveSupport::Multibyte::Unicode.default_normalization_form - def normalize(form = nil) - form ||= Unicode.default_normalization_form - - # See https://www.unicode.org/reports/tr15, Table 1 - if alias_form = Unicode::NORMALIZATION_FORM_ALIASES[form] - ActiveSupport::Deprecation.warn(<<-MSG.squish) - ActiveSupport::Multibyte::Chars#normalize is deprecated and will be - removed from Rails 6.1. Use #unicode_normalize(:#{alias_form}) instead. - MSG - - unicode_normalize(alias_form) - else - ActiveSupport::Deprecation.warn(<<-MSG.squish) - ActiveSupport::Multibyte::Chars#normalize is deprecated and will be - removed from Rails 6.1. Use #unicode_normalize instead. - MSG - - raise ArgumentError, "#{form} is not a valid normalization variant", caller - end - end - # Performs canonical decomposition on all the characters. # # 'é'.length # => 2 diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 1200c79b68..35560ede23 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -72,18 +72,6 @@ class MultibyteCharsTest < ActiveSupport::TestCase assert_equal "abb", mb_a << mb_b end - def test_consumes_utf8_strings - ActiveSupport::Deprecation.silence do - assert @proxy_class.consumes?(UNICODE_STRING) - assert @proxy_class.consumes?(ASCII_STRING) - assert_not @proxy_class.consumes?(BYTE_STRING) - end - end - - def test_consumes_is_deprecated - assert_deprecated { @proxy_class.consumes?(UNICODE_STRING) } - end - def test_concatenation_should_return_a_proxy_class_instance assert_equal ActiveSupport::Multibyte.proxy_class, ("a".mb_chars + "b").class assert_equal ActiveSupport::Multibyte.proxy_class, ((+"a").mb_chars << "b").class @@ -171,9 +159,6 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase assert chars("").upcase.kind_of?(ActiveSupport::Multibyte.proxy_class) assert chars("").downcase.kind_of?(ActiveSupport::Multibyte.proxy_class) assert chars("").capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class) - ActiveSupport::Deprecation.silence do - assert chars("").normalize.kind_of?(ActiveSupport::Multibyte.proxy_class) - end assert chars("").decompose.kind_of?(ActiveSupport::Multibyte.proxy_class) assert chars("").compose.kind_of?(ActiveSupport::Multibyte.proxy_class) assert chars("").tidy_bytes.kind_of?(ActiveSupport::Multibyte.proxy_class) @@ -397,12 +382,6 @@ class MultibyteCharsUTF8BehaviourTest < ActiveSupport::TestCase def test_reverse_should_work_with_normalized_strings str = "bös" reversed_str = "söb" - ActiveSupport::Deprecation.silence do - assert_equal chars(reversed_str).normalize(:kc), chars(str).normalize(:kc).reverse - assert_equal chars(reversed_str).normalize(:c), chars(str).normalize(:c).reverse - assert_equal chars(reversed_str).normalize(:d), chars(str).normalize(:d).reverse - assert_equal chars(reversed_str).normalize(:kd), chars(str).normalize(:kd).reverse - end assert_equal chars(reversed_str).decompose, chars(str).decompose.reverse assert_equal chars(reversed_str).compose, chars(str).compose.reverse end @@ -581,57 +560,13 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase end end - def test_composition_exclusion_is_set_up_properly - # Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly - qa = [0x915, 0x93c].pack("U*") - ActiveSupport::Deprecation.silence do - assert_equal qa, chars(qa).normalize(:c) - end - end - - # Test for the Public Review Issue #29, bad explanation of composition might lead to a - # bad implementation: http://www.unicode.org/review/pr-29.html - def test_normalization_C_pri_29 - [ - [0x0B47, 0x0300, 0x0B3E], - [0x1100, 0x0300, 0x1161] - ].map { |c| c.pack("U*") }.each do |c| - ActiveSupport::Deprecation.silence do - assert_equal_codepoints c, chars(c).normalize(:c) - end - end - end - def test_normalization_shouldnt_strip_null_bytes null_byte_str = "Test\0test" - ActiveSupport::Deprecation.silence do - assert_equal null_byte_str, chars(null_byte_str).normalize(:kc) - assert_equal null_byte_str, chars(null_byte_str).normalize(:c) - assert_equal null_byte_str, chars(null_byte_str).normalize(:d) - assert_equal null_byte_str, chars(null_byte_str).normalize(:kd) - end assert_equal null_byte_str, chars(null_byte_str).decompose assert_equal null_byte_str, chars(null_byte_str).compose end - def test_simple_normalization - comp_str = [ - 44, # LATIN CAPITAL LETTER D - 307, # COMBINING DOT ABOVE - 328, # COMBINING OGONEK - 323 # COMBINING DOT BELOW - ].pack("U*") - - ActiveSupport::Deprecation.silence do - assert_equal_codepoints "", chars("").normalize - assert_equal_codepoints [44, 105, 106, 328, 323].pack("U*"), chars(comp_str).normalize(:kc).to_s - assert_equal_codepoints [44, 307, 328, 323].pack("U*"), chars(comp_str).normalize(:c).to_s - assert_equal_codepoints [44, 307, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:d).to_s - assert_equal_codepoints [44, 105, 106, 110, 780, 78, 769].pack("U*"), chars(comp_str).normalize(:kd).to_s - end - end - def test_should_compute_grapheme_length [ ["", 0], @@ -757,21 +692,6 @@ class MultibyteCharsExtrasTest < ActiveSupport::TestCase end end - def test_chars_normalize_deprecation - # String#unicode_normalize default form is `:nfc`, and - # different than Multibyte::Unicode default, `:nkfc`. - # Deprecation should suggest the right form if no params - # are given and default is used. - assert_deprecated(/unicode_normalize\(:nfkc\)/) do - "".mb_chars.normalize - end - - assert_deprecated(/unicode_normalize\(:nfc\)/) { "".mb_chars.normalize(:c) } - assert_deprecated(/unicode_normalize\(:nfd\)/) { "".mb_chars.normalize(:d) } - assert_deprecated(/unicode_normalize\(:nfkc\)/) { "".mb_chars.normalize(:kc) } - assert_deprecated(/unicode_normalize\(:nfkd\)/) { "".mb_chars.normalize(:kd) } - end - def test_unicode_deprecations assert_deprecated { ActiveSupport::Multibyte::Unicode.downcase("") } assert_deprecated { ActiveSupport::Multibyte::Unicode.upcase("") } diff --git a/activesupport/test/multibyte_conformance_test.rb b/activesupport/test/multibyte_conformance_test.rb deleted file mode 100644 index 95dab84194..0000000000 --- a/activesupport/test/multibyte_conformance_test.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -require_relative "abstract_unit" -require_relative "multibyte_test_helpers" - -class MultibyteConformanceTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - UNIDATA_FILE = "/NormalizationTest.txt" - RUN_P = begin - Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE) - rescue - end - - def setup - @proxy = ActiveSupport::Multibyte::Chars - skip "Unable to download test data" unless RUN_P - end - - def test_normalizations_C - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - - # CONFORMANCE: - # 1. The following invariants must be true for all conformant implementations - # - # NFC - # c2 == NFC(c1) == NFC(c2) == NFC(c3) - assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}" - # - # c4 == NFC(c4) == NFC(c5) - assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}" - end - end - end - - def test_normalizations_D - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - # - # NFD - # c3 == NFD(c1) == NFD(c2) == NFD(c3) - assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}" - # c5 == NFD(c4) == NFD(c5) - assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}" - end - end - end - - def test_normalizations_KC - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKC - # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) - assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}" - end - end - end - - def test_normalizations_KD - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKD - # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) - assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}" - end - end - end - - private - def each_line_of_norm_tests(&block) - File.open(File.join(CACHE_DIR, UNIDATA_FILE), "r") do | f | - until f.eof? - line = f.gets.chomp! - next if line.empty? || line.start_with?("#") - - cols, comment = line.split("#") - cols = cols.split(";").map(&:strip).reject(&:empty?) - next unless cols.length == 5 - - # codepoints are in hex in the test suite, pack wants them as integers - cols.map! { |c| c.split.map { |codepoint| codepoint.to_i(16) }.pack("U*") } - cols << comment - - yield(*cols) - end - end - end - - def inspect_codepoints(str) - str.to_s.unpack("U*").map { |cp| cp.to_s(16) }.join(" ") - end -end diff --git a/activesupport/test/multibyte_normalization_conformance_test.rb b/activesupport/test/multibyte_normalization_conformance_test.rb deleted file mode 100644 index 581a1a32d1..0000000000 --- a/activesupport/test/multibyte_normalization_conformance_test.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -require_relative "abstract_unit" -require_relative "multibyte_test_helpers" - -class MultibyteNormalizationConformanceTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - UNIDATA_FILE = "/NormalizationTest.txt" - RUN_P = begin - Downloader.download(UNIDATA_URL + UNIDATA_FILE, CACHE_DIR + UNIDATA_FILE) - rescue - end - - def setup - @proxy = ActiveSupport::Multibyte::Chars - skip "Unable to download test data" unless RUN_P - end - - def test_normalizations_C - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - - # CONFORMANCE: - # 1. The following invariants must be true for all conformant implementations - # - # NFC - # c2 == NFC(c1) == NFC(c2) == NFC(c3) - assert_equal_codepoints col2, @proxy.new(col1).normalize(:c), "Form C - Col 2 has to be NFC(1) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col2).normalize(:c), "Form C - Col 2 has to be NFC(2) - #{comment}" - assert_equal_codepoints col2, @proxy.new(col3).normalize(:c), "Form C - Col 2 has to be NFC(3) - #{comment}" - # - # c4 == NFC(c4) == NFC(c5) - assert_equal_codepoints col4, @proxy.new(col4).normalize(:c), "Form C - Col 4 has to be C(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:c), "Form C - Col 4 has to be C(5) - #{comment}" - end - end - end - - def test_normalizations_D - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do |*cols| - col1, col2, col3, col4, col5, comment = *cols - # - # NFD - # c3 == NFD(c1) == NFD(c2) == NFD(c3) - assert_equal_codepoints col3, @proxy.new(col1).normalize(:d), "Form D - Col 3 has to be NFD(1) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col2).normalize(:d), "Form D - Col 3 has to be NFD(2) - #{comment}" - assert_equal_codepoints col3, @proxy.new(col3).normalize(:d), "Form D - Col 3 has to be NFD(3) - #{comment}" - # c5 == NFD(c4) == NFD(c5) - assert_equal_codepoints col5, @proxy.new(col4).normalize(:d), "Form D - Col 5 has to be NFD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:d), "Form D - Col 5 has to be NFD(5) - #{comment}" - end - end - end - - def test_normalizations_KC - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKC - # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) - assert_equal_codepoints col4, @proxy.new(col1).normalize(:kc), "Form D - Col 4 has to be NFKC(1) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col2).normalize(:kc), "Form D - Col 4 has to be NFKC(2) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col3).normalize(:kc), "Form D - Col 4 has to be NFKC(3) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col4).normalize(:kc), "Form D - Col 4 has to be NFKC(4) - #{comment}" - assert_equal_codepoints col4, @proxy.new(col5).normalize(:kc), "Form D - Col 4 has to be NFKC(5) - #{comment}" - end - end - end - - def test_normalizations_KD - ActiveSupport::Deprecation.silence do - each_line_of_norm_tests do | *cols | - col1, col2, col3, col4, col5, comment = *cols - # - # NFKD - # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) - assert_equal_codepoints col5, @proxy.new(col1).normalize(:kd), "Form KD - Col 5 has to be NFKD(1) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col2).normalize(:kd), "Form KD - Col 5 has to be NFKD(2) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col3).normalize(:kd), "Form KD - Col 5 has to be NFKD(3) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col4).normalize(:kd), "Form KD - Col 5 has to be NFKD(4) - #{comment}" - assert_equal_codepoints col5, @proxy.new(col5).normalize(:kd), "Form KD - Col 5 has to be NFKD(5) - #{comment}" - end - end - end - - private - def each_line_of_norm_tests(&block) - lines = 0 - max_test_lines = 0 # Don't limit below 38, because that's the header of the testfile - File.open(File.join(CACHE_DIR, UNIDATA_FILE), "r") do | f | - until f.eof? || (max_test_lines > 38 && lines > max_test_lines) - lines += 1 - line = f.gets.chomp! - next if line.empty? || line.start_with?("#") - - cols, comment = line.split("#") - cols = cols.split(";").map { |e| e.strip }.reject { |e| e.empty? } - next unless cols.length == 5 - - # codepoints are in hex in the test suite, pack wants them as integers - cols.map! { |c| c.split.map { |codepoint| codepoint.to_i(16) }.pack("U*") } - cols << comment - - yield(*cols) - end - end - end - - def inspect_codepoints(str) - str.to_s.unpack("U*").map { |cp| cp.to_s(16) }.join(" ") - end -end diff --git a/guides/source/6_1_release_notes.md b/guides/source/6_1_release_notes.md index 7ac80238f3..6f6e073f39 100644 --- a/guides/source/6_1_release_notes.md +++ b/guides/source/6_1_release_notes.md @@ -304,6 +304,8 @@ Please refer to the [Changelog][active-support] for detailed changes. * Remove deprecated file `active_support/core_ext/range/include_range`. +* Remove deprecated `ActiveSupport::Multibyte::Chars#consumes?` and `ActiveSupport::Multibyte::Chars#normalize`. + ### Deprecations ### Notable changes