diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index 57e8e24bf4..5efe13c537 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -1,9 +1,5 @@ -# encoding: utf-8 -require 'active_support/core_ext/module/attribute_accessors' - module ActiveSupport #:nodoc: module Multibyte - autoload :EncodingError, 'active_support/multibyte/exceptions' autoload :Chars, 'active_support/multibyte/chars' autoload :Unicode, 'active_support/multibyte/unicode' @@ -21,24 +17,5 @@ module ActiveSupport #:nodoc: def self.proxy_class @proxy_class ||= ActiveSupport::Multibyte::Chars end - - # Regular expressions that describe valid byte sequences for a character - VALID_CHARACTER = { - # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site) - 'UTF-8' => /\A(?: - [\x00-\x7f] | - [\xc2-\xdf] [\x80-\xbf] | - \xe0 [\xa0-\xbf] [\x80-\xbf] | - [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] | - \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] | - [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] | - \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn, - # Quick check for valid Shift-JIS characters, disregards the odd-even pairing - 'Shift_JIS' => /\A(?: - [\x00-\x7e\xa1-\xdf] | - [\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn - } end -end - -require 'active_support/multibyte/utils' \ No newline at end of file +end \ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index dcc176e93f..99b974e4a7 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require 'active_support/core_ext/string/access' require 'active_support/core_ext/string/behavior' +require 'active_support/core_ext/module/delegation' module ActiveSupport #:nodoc: module Multibyte #:nodoc: @@ -34,10 +35,13 @@ module ActiveSupport #:nodoc: # # ActiveSupport::Multibyte.proxy_class = CharsForUTF32 class Chars + include Comparable attr_reader :wrapped_string alias to_s wrapped_string alias to_str wrapped_string + delegate :<=>, :=~, :acts_like_string?, :to => :wrapped_string + # Creates a new Chars instance by wrapping _string_. def initialize(string) @wrapped_string = string @@ -47,8 +51,8 @@ module ActiveSupport #:nodoc: # Forward all undefined methods to the wrapped string. def method_missing(method, *args, &block) if method.to_s =~ /!$/ - @wrapped_string.__send__(method, *args, &block) - self + result = @wrapped_string.__send__(method, *args, &block) + self if result else result = @wrapped_string.__send__(method, *args, &block) result.kind_of?(String) ? chars(result) : result @@ -61,35 +65,9 @@ module ActiveSupport #:nodoc: super || @wrapped_string.respond_to?(method, include_private) end - # Enable more predictable duck-typing on String-like classes. See Object#acts_like?. - def acts_like_string? - true - end - # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise. def self.consumes?(string) - # Unpack is a little bit faster than regular expressions. - string.unpack('U*') - true - rescue ArgumentError - false - end - - include Comparable - - # Returns -1, 0, or 1, depending on whether the Chars object is to be sorted before, - # equal or after the object on the right side of the operation. It accepts any object - # that implements +to_s+: - # - # 'é'.mb_chars <=> 'ü'.mb_chars # => -1 - # - # See String#<=> for more details. - def <=>(other) - @wrapped_string <=> other.to_s - end - - def =~(other) - @wrapped_string =~ other + string.encoding == Encoding::UTF_8 end # Works just like String#split, with the exception that the items in the resulting list are Chars @@ -101,45 +79,10 @@ module ActiveSupport #:nodoc: @wrapped_string.split(*args).map { |i| i.mb_chars } end - # Like String#[]=, except instead of byte offsets you specify character offsets. - # - # Example: - # - # s = "Müller" - # s.mb_chars[2] = "e" # Replace character with offset 2 - # s - # # => "Müeler" - # - # s = "Müller" - # s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1 - # s - # # => "Möler" - def []=(*args) - replace_by = args.pop - # Indexed replace with regular expressions already works - if args.first.is_a?(Regexp) - @wrapped_string[*args] = replace_by - else - result = Unicode.u_unpack(@wrapped_string) - case args.first - when Fixnum - raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length - min = args[0] - max = args[1].nil? ? min : (min + args[1] - 1) - range = Range.new(min, max) - replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum) - when Range - raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length - range = args[0] - else - needle = args[0].to_s - min = index(needle) - max = min + Unicode.u_unpack(needle).length - 1 - range = Range.new(min, max) - end - result[range] = Unicode.u_unpack(replace_by) - @wrapped_string.replace(result.pack('U*')) - end + # Works like like String#slice!, but returns an instance of Chars, or nil if the string was not + # modified. + def slice!(*args) + chars(@wrapped_string.slice!(*args)) end # Reverses all characters in the string. @@ -147,37 +90,9 @@ module ActiveSupport #:nodoc: # Example: # 'Café'.mb_chars.reverse.to_s # => 'éfaC' def reverse - chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) + chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack('U*')) end - # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that - # character. - # - # Example: - # 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち" - def slice(*args) - if args.size > 2 - raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native - elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp))) - raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native - elsif (args.size == 2 && !args[1].is_a?(Numeric)) - raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native - elsif args[0].kind_of? Range - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps.nil? ? nil : cps.pack('U*') - elsif args[0].kind_of? Regexp - result = @wrapped_string.slice(*args) - elsif args.size == 1 && args[0].kind_of?(Numeric) - character = Unicode.u_unpack(@wrapped_string)[args[0]] - result = character && [character].pack('U') - else - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps && cps.pack('U*') - end - result && chars(result) - end - alias_method :[], :slice - # Limit the byte size of the string to a number of bytes without breaking characters. Usable # when the storage for a string is limited for some reason. # @@ -192,7 +107,7 @@ module ActiveSupport #:nodoc: # Example: # 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?" def upcase - chars(Unicode.apply_mapping @wrapped_string, :uppercase_mapping) + chars Unicode.upcase(@wrapped_string) end # Convert characters in the string to lowercase. @@ -200,7 +115,7 @@ module ActiveSupport #:nodoc: # Example: # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum" def downcase - chars(Unicode.apply_mapping @wrapped_string, :lowercase_mapping) + chars Unicode.downcase(@wrapped_string) end # Converts the first character to uppercase and the remainder to lowercase. @@ -217,7 +132,7 @@ module ActiveSupport #:nodoc: # "ÉL QUE SE ENTERÓ".mb_chars.titleize # => "Él Que Se Enteró" # "日本語".mb_chars.titleize # => "日本語" def titleize - chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.apply_mapping $1, :uppercase_mapping }) + chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.upcase($1)}) end alias_method :titlecase, :titleize @@ -237,7 +152,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 2 # 'é'.mb_chars.decompose.to_s.length # => 3 def decompose - chars(Unicode.decompose_codepoints(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack('U*')) end # Performs composition on all the characters. @@ -246,16 +161,16 @@ module ActiveSupport #:nodoc: # 'é'.length # => 3 # 'é'.mb_chars.compose.to_s.length # => 2 def compose - chars(Unicode.compose_codepoints(Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack('U*')) end # Returns the number of grapheme clusters in the string. # # Example: # 'क्षि'.mb_chars.length # => 4 - # 'क्षि'.mb_chars.g_length # => 3 - def g_length - Unicode.g_unpack(@wrapped_string).length + # 'क्षि'.mb_chars.grapheme_length # => 3 + def grapheme_length + Unicode.unpack_graphemes(@wrapped_string).length end # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string. @@ -265,14 +180,10 @@ module ActiveSupport #:nodoc: chars(Unicode.tidy_bytes(@wrapped_string, force)) end - %w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method| - # Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will - # exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings. - if public_method_defined?(method) - define_method("#{method}!") do |*args| - @wrapped_string = send(args.nil? ? method : method, *args).to_s - self - end + %w(capitalize downcase reverse tidy_bytes upcase).each do |method| + define_method("#{method}!") do |*args| + @wrapped_string = send(method, *args).to_s + self end end @@ -282,41 +193,14 @@ module ActiveSupport #:nodoc: return nil if byte_offset.nil? return 0 if @wrapped_string == '' - @wrapped_string = @wrapped_string.dup.force_encoding(Encoding::ASCII_8BIT) - begin - @wrapped_string[0...byte_offset].unpack('U*').length + @wrapped_string.byteslice(0...byte_offset).unpack('U*').length rescue ArgumentError byte_offset -= 1 retry end end - def justify(integer, way, padstr=' ') #:nodoc: - raise ArgumentError, "zero width padding" if padstr.length == 0 - padsize = integer - size - padsize = padsize > 0 ? padsize : 0 - case way - when :right - result = @wrapped_string.dup.insert(0, padding(padsize, padstr)) - when :left - result = @wrapped_string.dup.insert(-1, padding(padsize, padstr)) - when :center - lpad = padding((padsize / 2.0).floor, padstr) - rpad = padding((padsize / 2.0).ceil, padstr) - result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad) - end - chars(result) - end - - def padding(padsize, padstr=' ') #:nodoc: - if padsize != 0 - chars(padstr * ((padsize / Unicode.u_unpack(padstr).size) + 1)).slice(0, padsize) - else - '' - end - end - def chars(string) #:nodoc: self.class.new(string) end diff --git a/activesupport/lib/active_support/multibyte/exceptions.rb b/activesupport/lib/active_support/multibyte/exceptions.rb deleted file mode 100644 index 62066e3c71..0000000000 --- a/activesupport/lib/active_support/multibyte/exceptions.rb +++ /dev/null @@ -1,8 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Raised when a problem with the encoding was found. - class EncodingError < StandardError; end - end -end \ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 754ca9290b..94fb0a48aa 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -10,7 +10,7 @@ module ActiveSupport NORMALIZATION_FORMS = [:c, :kc, :d, :kd] # The Unicode version that is supported by the implementation - UNICODE_VERSION = '5.2.0' + UNICODE_VERSION = '6.0.0' # The default normalization used for operations that require normalization. It can be set to any of the # normalizations in NORMALIZATION_FORMS. @@ -61,19 +61,6 @@ module ActiveSupport TRAILERS_PAT = /(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u LEADERS_PAT = /\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u - # Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't - # valid UTF-8. - # - # Example: - # Unicode.u_unpack('Café') # => [67, 97, 102, 233] - def u_unpack(string) - begin - string.unpack 'U*' - rescue ArgumentError - raise EncodingError, 'malformed UTF-8 character' - end - end - # Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified # character class and +false+ otherwise. Valid character classes are: :cr, :lf, :l, # :v, :lv, :lvt and :t. @@ -86,10 +73,10 @@ module ActiveSupport # Unpack the string at grapheme boundaries. Returns a list of character lists. # # Example: - # Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]] - # Unicode.g_unpack('Café') # => [[67], [97], [102], [233]] - def g_unpack(string) - codepoints = u_unpack(string) + # Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]] + # Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]] + def unpack_graphemes(string) + codepoints = string.codepoints.to_a unpacked = [] pos = 0 marker = 0 @@ -118,12 +105,12 @@ module ActiveSupport unpacked end - # Reverse operation of g_unpack. + # Reverse operation of unpack_graphemes. # # Example: - # Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि' - def g_pack(unpacked) - (unpacked.flatten).pack('U*') + # Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि' + def pack_graphemes(unpacked) + unpacked.flatten.pack('U*') end # Re-order codepoints so the string becomes canonical. @@ -143,7 +130,7 @@ module ActiveSupport end # Decompose composed characters to the decomposed form. - def decompose_codepoints(type, codepoints) + def decompose(type, codepoints) codepoints.inject([]) do |decomposed, cp| # if it's a hangul syllable starter character if HANGUL_SBASE <= cp and cp < HANGUL_SLAST @@ -156,7 +143,7 @@ module ActiveSupport decomposed.concat ncp # if the codepoint is decomposable in with the current decomposition type elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability) - decomposed.concat decompose_codepoints(type, ncp.dup) + decomposed.concat decompose(type, ncp.dup) else decomposed << cp end @@ -164,7 +151,7 @@ module ActiveSupport end # Compose decomposed characters to the composed form. - def compose_codepoints(codepoints) + def compose(codepoints) pos = 0 eoa = codepoints.length - 1 starter_pos = 0 @@ -283,30 +270,27 @@ module ActiveSupport def normalize(string, form=nil) form ||= @default_normalization_form # See http://www.unicode.org/reports/tr15, Table 1 - codepoints = u_unpack(string) + codepoints = string.codepoints.to_a case form when :d - reorder_characters(decompose_codepoints(:canonical, codepoints)) + reorder_characters(decompose(:canonical, codepoints)) when :c - compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints))) + compose(reorder_characters(decompose(:canonical, codepoints))) when :kd - reorder_characters(decompose_codepoints(:compatability, codepoints)) + reorder_characters(decompose(:compatability, codepoints)) when :kc - compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints))) + compose(reorder_characters(decompose(:compatability, codepoints))) else raise ArgumentError, "#{form} is not a valid normalization variant", caller end.pack('U*') end - def apply_mapping(string, mapping) #:nodoc: - u_unpack(string).map do |codepoint| - cp = database.codepoints[codepoint] - if cp and (ncp = cp.send(mapping)) and ncp > 0 - ncp - else - codepoint - end - end.pack('U*') + def downcase(string) + apply_mapping string, :lowercase_mapping + end + + def upcase(string) + apply_mapping string, :uppercase_mapping end # Holds data about a codepoint in the Unicode database @@ -374,6 +358,17 @@ module ActiveSupport private + def apply_mapping(string, mapping) #:nodoc: + string.each_codepoint.map do |codepoint| + cp = database.codepoints[codepoint] + if cp and (ncp = cp.send(mapping)) and ncp > 0 + ncp + else + codepoint + end + end.pack('U*') + end + def tidy_byte(byte) if byte < 160 [database.cp1252[byte] || byte].pack("U").unpack("C*") diff --git a/activesupport/lib/active_support/multibyte/utils.rb b/activesupport/lib/active_support/multibyte/utils.rb deleted file mode 100644 index bd6d4bad41..0000000000 --- a/activesupport/lib/active_support/multibyte/utils.rb +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Returns a regular expression that matches valid characters in the current encoding - def self.valid_character - VALID_CHARACTER[Encoding.default_external.to_s] - end - - # Verifies the encoding of a string - def self.verify(string) - string.valid_encoding? - end - - # Verifies the encoding of the string and raises an exception when it's not valid - def self.verify!(string) - raise EncodingError.new("Found characters with invalid encoding") unless verify(string) - end - - # Removes all invalid characters from the string. - # - # Note: this method is a no-op in Ruby 1.9 - def self.clean(string) - string - end - end -end diff --git a/activesupport/lib/active_support/values/unicode_tables.dat b/activesupport/lib/active_support/values/unicode_tables.dat index 4fe0268cca..7edc4663e8 100644 Binary files a/activesupport/lib/active_support/values/unicode_tables.dat and b/activesupport/lib/active_support/values/unicode_tables.dat differ diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 20e56e2c81..e36bd22081 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -7,6 +7,7 @@ class String def __method_for_multibyte_testing_with_integer_result; 1; end def __method_for_multibyte_testing; 'result'; end def __method_for_multibyte_testing!; 'result'; end + def __method_for_multibyte_testing_that_returns_nil!; end end class MultibyteCharsTest < Test::Unit::TestCase @@ -36,11 +37,15 @@ class MultibyteCharsTest < Test::Unit::TestCase assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id end - def test_forwarded_bang_method_calls_should_return_the_original_chars_instance + def test_forwarded_bang_method_calls_should_return_the_original_chars_instance_when_result_is_not_nil assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing! assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id end + def test_forwarded_bang_method_calls_should_return_nil_when_result_is_nil + assert_nil @chars.__method_for_multibyte_testing_that_returns_nil! + end + def test_methods_are_forwarded_to_wrapped_string_for_byte_strings assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class end @@ -67,17 +72,6 @@ class MultibyteCharsTest < Test::Unit::TestCase assert !@proxy_class.consumes?(BYTE_STRING) end - def test_unpack_utf8_strings - assert_equal 4, ActiveSupport::Multibyte::Unicode.u_unpack(UNICODE_STRING).length - assert_equal 5, ActiveSupport::Multibyte::Unicode.u_unpack(ASCII_STRING).length - end - - def test_unpack_raises_encoding_error_on_broken_strings - assert_raise(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte::Unicode.u_unpack(BYTE_STRING) - end - end - def test_concatenation_should_return_a_proxy_class_instance assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class @@ -112,15 +106,11 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase end end - def test_indexed_insert_accepts_fixnums - @chars[2] = 32 - assert_equal 'こに わ', @chars - end - - %w{capitalize downcase lstrip reverse rstrip strip upcase}.each do |method| + %w{capitalize downcase lstrip reverse rstrip upcase}.each do |method| class_eval(<<-EOTESTS) - def test_#{method}_bang_should_return_self - assert_equal @chars.object_id, @chars.send("#{method}!").object_id + def test_#{method}_bang_should_return_self_when_modifying_wrapped_string + chars = ' él piDió Un bUen café ' + assert_equal chars.object_id, chars.send("#{method}!").object_id end def test_#{method}_bang_should_change_wrapped_string @@ -419,7 +409,7 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase def test_slice_bang_removes_the_slice_from_the_receiver chars = 'úüù'.mb_chars chars.slice!(0,2) - assert_equal 'úü', chars + assert_equal 'ù', chars end def test_slice_should_throw_exceptions_on_invalid_arguments @@ -506,7 +496,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase def test_limit_should_work_on_a_multibyte_string example = chars(UNICODE_STRING) - bytesize = UNICODE_STRING.respond_to?(:bytesize) ? UNICODE_STRING.bytesize : UNICODE_STRING.size + bytesize = UNICODE_STRING.bytesize assert_equal UNICODE_STRING, example.limit(bytesize) assert_equal '', example.limit(0) @@ -605,7 +595,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase else str = input end - assert_equal expected_length, chars(str).g_length + assert_equal expected_length, chars(str).grapheme_length end end diff --git a/activesupport/test/multibyte_utils_test.rb b/activesupport/test/multibyte_utils_test.rb deleted file mode 100644 index f807492be0..0000000000 --- a/activesupport/test/multibyte_utils_test.rb +++ /dev/null @@ -1,93 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' -require 'multibyte_test_helpers' - -class MultibyteUtilsTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - test "valid_character returns an expression for the current encoding" do - with_encoding('None') do - assert_nil ActiveSupport::Multibyte.valid_character - end - with_encoding('UTF8') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character - end - with_encoding('SJIS') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character - end - end - - test "verify verifies ASCII strings are properly encoded" do - with_encoding('None') do - examples.each do |example| - assert ActiveSupport::Multibyte.verify(example) - end - end - end - - test "verify verifies UTF-8 strings are properly encoded" do - with_encoding('UTF8') do - assert ActiveSupport::Multibyte.verify(example('valid UTF-8')) - assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8')) - end - end - - test "verify verifies Shift-JIS strings are properly encoded" do - with_encoding('SJIS') do - assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS')) - assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS')) - end - end - - test "verify! raises an exception when it finds an invalid character" do - with_encoding('UTF8') do - assert_raises(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte.verify!(example('invalid UTF-8')) - end - end - end - - test "verify! doesn't raise an exception when the encoding is valid" do - with_encoding('UTF8') do - assert_nothing_raised do - ActiveSupport::Multibyte.verify!(example('valid UTF-8')) - end - end - end - - test "clean is a no-op" do - with_encoding('UTF8') do - assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) - end - end - - private - - STRINGS = { - 'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'), - 'invalid ASCII' => [128].pack('C*'), - 'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'), - 'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'), - 'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'), - 'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*') - } - - def example(key) - STRINGS[key].force_encoding(Encoding.default_external) - end - - def examples - STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) } - end - - KCODE_TO_ENCODING = Hash.new(Encoding::BINARY). - update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS) - - def with_encoding(enc) - before = Encoding.default_external - silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] } - yield - silence_warnings { Encoding.default_external = before } - end -end