Merge pull request #4332 from norman/multibyte
Putting AS::Multibyte on a Ruby 1.9 diet
This commit is contained in:
José Valim
commit
4751cc21e8
|
|
@ -1,9 +1,5 @@
|
|||
# encoding: utf-8
|
||||
require 'active_support/core_ext/module/attribute_accessors'
|
||||
|
||||
module ActiveSupport #:nodoc:
|
||||
module Multibyte
|
||||
autoload :EncodingError, 'active_support/multibyte/exceptions'
|
||||
autoload :Chars, 'active_support/multibyte/chars'
|
||||
autoload :Unicode, 'active_support/multibyte/unicode'
|
||||
|
||||
|
|
@ -21,24 +17,5 @@ module ActiveSupport #:nodoc:
|
|||
def self.proxy_class
|
||||
@proxy_class ||= ActiveSupport::Multibyte::Chars
|
||||
end
|
||||
|
||||
# Regular expressions that describe valid byte sequences for a character
|
||||
VALID_CHARACTER = {
|
||||
# Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
|
||||
'UTF-8' => /\A(?:
|
||||
[\x00-\x7f] |
|
||||
[\xc2-\xdf] [\x80-\xbf] |
|
||||
\xe0 [\xa0-\xbf] [\x80-\xbf] |
|
||||
[\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
|
||||
\xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
|
||||
[\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
|
||||
\xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn,
|
||||
# Quick check for valid Shift-JIS characters, disregards the odd-even pairing
|
||||
'Shift_JIS' => /\A(?:
|
||||
[\x00-\x7e\xa1-\xdf] |
|
||||
[\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
require 'active_support/multibyte/utils'
|
||||
end
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
# encoding: utf-8
|
||||
require 'active_support/core_ext/string/access'
|
||||
require 'active_support/core_ext/string/behavior'
|
||||
require 'active_support/core_ext/module/delegation'
|
||||
|
||||
module ActiveSupport #:nodoc:
|
||||
module Multibyte #:nodoc:
|
||||
|
|
@ -34,10 +35,13 @@ module ActiveSupport #:nodoc:
|
|||
#
|
||||
# ActiveSupport::Multibyte.proxy_class = CharsForUTF32
|
||||
class Chars
|
||||
include Comparable
|
||||
attr_reader :wrapped_string
|
||||
alias to_s wrapped_string
|
||||
alias to_str wrapped_string
|
||||
|
||||
delegate :<=>, :=~, :acts_like_string?, :to => :wrapped_string
|
||||
|
||||
# Creates a new Chars instance by wrapping _string_.
|
||||
def initialize(string)
|
||||
@wrapped_string = string
|
||||
|
|
@ -47,8 +51,8 @@ module ActiveSupport #:nodoc:
|
|||
# Forward all undefined methods to the wrapped string.
|
||||
def method_missing(method, *args, &block)
|
||||
if method.to_s =~ /!$/
|
||||
@wrapped_string.__send__(method, *args, &block)
|
||||
self
|
||||
result = @wrapped_string.__send__(method, *args, &block)
|
||||
self if result
|
||||
else
|
||||
result = @wrapped_string.__send__(method, *args, &block)
|
||||
result.kind_of?(String) ? chars(result) : result
|
||||
|
|
@ -61,35 +65,9 @@ module ActiveSupport #:nodoc:
|
|||
super || @wrapped_string.respond_to?(method, include_private)
|
||||
end
|
||||
|
||||
# Enable more predictable duck-typing on String-like classes. See Object#acts_like?.
|
||||
def acts_like_string?
|
||||
true
|
||||
end
|
||||
|
||||
# Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise.
|
||||
def self.consumes?(string)
|
||||
# Unpack is a little bit faster than regular expressions.
|
||||
string.unpack('U*')
|
||||
true
|
||||
rescue ArgumentError
|
||||
false
|
||||
end
|
||||
|
||||
include Comparable
|
||||
|
||||
# Returns -1, 0, or 1, depending on whether the Chars object is to be sorted before,
|
||||
# equal or after the object on the right side of the operation. It accepts any object
|
||||
# that implements +to_s+:
|
||||
#
|
||||
# 'é'.mb_chars <=> 'ü'.mb_chars # => -1
|
||||
#
|
||||
# See <tt>String#<=></tt> for more details.
|
||||
def <=>(other)
|
||||
@wrapped_string <=> other.to_s
|
||||
end
|
||||
|
||||
def =~(other)
|
||||
@wrapped_string =~ other
|
||||
string.encoding == Encoding::UTF_8
|
||||
end
|
||||
|
||||
# Works just like <tt>String#split</tt>, with the exception that the items in the resulting list are Chars
|
||||
|
|
@ -101,45 +79,10 @@ module ActiveSupport #:nodoc:
|
|||
@wrapped_string.split(*args).map { |i| i.mb_chars }
|
||||
end
|
||||
|
||||
# Like <tt>String#[]=</tt>, except instead of byte offsets you specify character offsets.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# s = "Müller"
|
||||
# s.mb_chars[2] = "e" # Replace character with offset 2
|
||||
# s
|
||||
# # => "Müeler"
|
||||
#
|
||||
# s = "Müller"
|
||||
# s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1
|
||||
# s
|
||||
# # => "Möler"
|
||||
def []=(*args)
|
||||
replace_by = args.pop
|
||||
# Indexed replace with regular expressions already works
|
||||
if args.first.is_a?(Regexp)
|
||||
@wrapped_string[*args] = replace_by
|
||||
else
|
||||
result = Unicode.u_unpack(@wrapped_string)
|
||||
case args.first
|
||||
when Fixnum
|
||||
raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length
|
||||
min = args[0]
|
||||
max = args[1].nil? ? min : (min + args[1] - 1)
|
||||
range = Range.new(min, max)
|
||||
replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum)
|
||||
when Range
|
||||
raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length
|
||||
range = args[0]
|
||||
else
|
||||
needle = args[0].to_s
|
||||
min = index(needle)
|
||||
max = min + Unicode.u_unpack(needle).length - 1
|
||||
range = Range.new(min, max)
|
||||
end
|
||||
result[range] = Unicode.u_unpack(replace_by)
|
||||
@wrapped_string.replace(result.pack('U*'))
|
||||
end
|
||||
# Works like like <tt>String#slice!</tt>, but returns an instance of Chars, or nil if the string was not
|
||||
# modified.
|
||||
def slice!(*args)
|
||||
chars(@wrapped_string.slice!(*args))
|
||||
end
|
||||
|
||||
# Reverses all characters in the string.
|
||||
|
|
@ -147,37 +90,9 @@ module ActiveSupport #:nodoc:
|
|||
# Example:
|
||||
# 'Café'.mb_chars.reverse.to_s # => 'éfaC'
|
||||
def reverse
|
||||
chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*'))
|
||||
chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack('U*'))
|
||||
end
|
||||
|
||||
# Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that
|
||||
# character.
|
||||
#
|
||||
# Example:
|
||||
# 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち"
|
||||
def slice(*args)
|
||||
if args.size > 2
|
||||
raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native
|
||||
elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp)))
|
||||
raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native
|
||||
elsif (args.size == 2 && !args[1].is_a?(Numeric))
|
||||
raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native
|
||||
elsif args[0].kind_of? Range
|
||||
cps = Unicode.u_unpack(@wrapped_string).slice(*args)
|
||||
result = cps.nil? ? nil : cps.pack('U*')
|
||||
elsif args[0].kind_of? Regexp
|
||||
result = @wrapped_string.slice(*args)
|
||||
elsif args.size == 1 && args[0].kind_of?(Numeric)
|
||||
character = Unicode.u_unpack(@wrapped_string)[args[0]]
|
||||
result = character && [character].pack('U')
|
||||
else
|
||||
cps = Unicode.u_unpack(@wrapped_string).slice(*args)
|
||||
result = cps && cps.pack('U*')
|
||||
end
|
||||
result && chars(result)
|
||||
end
|
||||
alias_method :[], :slice
|
||||
|
||||
# Limit the byte size of the string to a number of bytes without breaking characters. Usable
|
||||
# when the storage for a string is limited for some reason.
|
||||
#
|
||||
|
|
@ -192,7 +107,7 @@ module ActiveSupport #:nodoc:
|
|||
# Example:
|
||||
# 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?"
|
||||
def upcase
|
||||
chars(Unicode.apply_mapping @wrapped_string, :uppercase_mapping)
|
||||
chars Unicode.upcase(@wrapped_string)
|
||||
end
|
||||
|
||||
# Convert characters in the string to lowercase.
|
||||
|
|
@ -200,7 +115,7 @@ module ActiveSupport #:nodoc:
|
|||
# Example:
|
||||
# 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum"
|
||||
def downcase
|
||||
chars(Unicode.apply_mapping @wrapped_string, :lowercase_mapping)
|
||||
chars Unicode.downcase(@wrapped_string)
|
||||
end
|
||||
|
||||
# Converts the first character to uppercase and the remainder to lowercase.
|
||||
|
|
@ -217,7 +132,7 @@ module ActiveSupport #:nodoc:
|
|||
# "ÉL QUE SE ENTERÓ".mb_chars.titleize # => "Él Que Se Enteró"
|
||||
# "日本語".mb_chars.titleize # => "日本語"
|
||||
def titleize
|
||||
chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.apply_mapping $1, :uppercase_mapping })
|
||||
chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.upcase($1)})
|
||||
end
|
||||
alias_method :titlecase, :titleize
|
||||
|
||||
|
|
@ -237,7 +152,7 @@ module ActiveSupport #:nodoc:
|
|||
# 'é'.length # => 2
|
||||
# 'é'.mb_chars.decompose.to_s.length # => 3
|
||||
def decompose
|
||||
chars(Unicode.decompose_codepoints(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*'))
|
||||
chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack('U*'))
|
||||
end
|
||||
|
||||
# Performs composition on all the characters.
|
||||
|
|
@ -246,16 +161,16 @@ module ActiveSupport #:nodoc:
|
|||
# 'é'.length # => 3
|
||||
# 'é'.mb_chars.compose.to_s.length # => 2
|
||||
def compose
|
||||
chars(Unicode.compose_codepoints(Unicode.u_unpack(@wrapped_string)).pack('U*'))
|
||||
chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack('U*'))
|
||||
end
|
||||
|
||||
# Returns the number of grapheme clusters in the string.
|
||||
#
|
||||
# Example:
|
||||
# 'क्षि'.mb_chars.length # => 4
|
||||
# 'क्षि'.mb_chars.g_length # => 3
|
||||
def g_length
|
||||
Unicode.g_unpack(@wrapped_string).length
|
||||
# 'क्षि'.mb_chars.grapheme_length # => 3
|
||||
def grapheme_length
|
||||
Unicode.unpack_graphemes(@wrapped_string).length
|
||||
end
|
||||
|
||||
# Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.
|
||||
|
|
@ -265,14 +180,10 @@ module ActiveSupport #:nodoc:
|
|||
chars(Unicode.tidy_bytes(@wrapped_string, force))
|
||||
end
|
||||
|
||||
%w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method|
|
||||
# Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will
|
||||
# exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings.
|
||||
if public_method_defined?(method)
|
||||
define_method("#{method}!") do |*args|
|
||||
@wrapped_string = send(args.nil? ? method : method, *args).to_s
|
||||
self
|
||||
end
|
||||
%w(capitalize downcase reverse tidy_bytes upcase).each do |method|
|
||||
define_method("#{method}!") do |*args|
|
||||
@wrapped_string = send(method, *args).to_s
|
||||
self
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -282,41 +193,14 @@ module ActiveSupport #:nodoc:
|
|||
return nil if byte_offset.nil?
|
||||
return 0 if @wrapped_string == ''
|
||||
|
||||
@wrapped_string = @wrapped_string.dup.force_encoding(Encoding::ASCII_8BIT)
|
||||
|
||||
begin
|
||||
@wrapped_string[0...byte_offset].unpack('U*').length
|
||||
@wrapped_string.byteslice(0...byte_offset).unpack('U*').length
|
||||
rescue ArgumentError
|
||||
byte_offset -= 1
|
||||
retry
|
||||
end
|
||||
end
|
||||
|
||||
def justify(integer, way, padstr=' ') #:nodoc:
|
||||
raise ArgumentError, "zero width padding" if padstr.length == 0
|
||||
padsize = integer - size
|
||||
padsize = padsize > 0 ? padsize : 0
|
||||
case way
|
||||
when :right
|
||||
result = @wrapped_string.dup.insert(0, padding(padsize, padstr))
|
||||
when :left
|
||||
result = @wrapped_string.dup.insert(-1, padding(padsize, padstr))
|
||||
when :center
|
||||
lpad = padding((padsize / 2.0).floor, padstr)
|
||||
rpad = padding((padsize / 2.0).ceil, padstr)
|
||||
result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad)
|
||||
end
|
||||
chars(result)
|
||||
end
|
||||
|
||||
def padding(padsize, padstr=' ') #:nodoc:
|
||||
if padsize != 0
|
||||
chars(padstr * ((padsize / Unicode.u_unpack(padstr).size) + 1)).slice(0, padsize)
|
||||
else
|
||||
''
|
||||
end
|
||||
end
|
||||
|
||||
def chars(string) #:nodoc:
|
||||
self.class.new(string)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
# encoding: utf-8
|
||||
|
||||
module ActiveSupport #:nodoc:
|
||||
module Multibyte #:nodoc:
|
||||
# Raised when a problem with the encoding was found.
|
||||
class EncodingError < StandardError; end
|
||||
end
|
||||
end
|
||||
|
|
@ -10,7 +10,7 @@ module ActiveSupport
|
|||
NORMALIZATION_FORMS = [:c, :kc, :d, :kd]
|
||||
|
||||
# The Unicode version that is supported by the implementation
|
||||
UNICODE_VERSION = '5.2.0'
|
||||
UNICODE_VERSION = '6.0.0'
|
||||
|
||||
# The default normalization used for operations that require normalization. It can be set to any of the
|
||||
# normalizations in NORMALIZATION_FORMS.
|
||||
|
|
@ -61,19 +61,6 @@ module ActiveSupport
|
|||
TRAILERS_PAT = /(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u
|
||||
LEADERS_PAT = /\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u
|
||||
|
||||
# Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't
|
||||
# valid UTF-8.
|
||||
#
|
||||
# Example:
|
||||
# Unicode.u_unpack('Café') # => [67, 97, 102, 233]
|
||||
def u_unpack(string)
|
||||
begin
|
||||
string.unpack 'U*'
|
||||
rescue ArgumentError
|
||||
raise EncodingError, 'malformed UTF-8 character'
|
||||
end
|
||||
end
|
||||
|
||||
# Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified
|
||||
# character class and +false+ otherwise. Valid character classes are: <tt>:cr</tt>, <tt>:lf</tt>, <tt>:l</tt>,
|
||||
# <tt>:v</tt>, <tt>:lv</tt>, <tt>:lvt</tt> and <tt>:t</tt>.
|
||||
|
|
@ -86,10 +73,10 @@ module ActiveSupport
|
|||
# Unpack the string at grapheme boundaries. Returns a list of character lists.
|
||||
#
|
||||
# Example:
|
||||
# Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]]
|
||||
# Unicode.g_unpack('Café') # => [[67], [97], [102], [233]]
|
||||
def g_unpack(string)
|
||||
codepoints = u_unpack(string)
|
||||
# Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]]
|
||||
# Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]]
|
||||
def unpack_graphemes(string)
|
||||
codepoints = string.codepoints.to_a
|
||||
unpacked = []
|
||||
pos = 0
|
||||
marker = 0
|
||||
|
|
@ -118,12 +105,12 @@ module ActiveSupport
|
|||
unpacked
|
||||
end
|
||||
|
||||
# Reverse operation of g_unpack.
|
||||
# Reverse operation of unpack_graphemes.
|
||||
#
|
||||
# Example:
|
||||
# Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि'
|
||||
def g_pack(unpacked)
|
||||
(unpacked.flatten).pack('U*')
|
||||
# Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि'
|
||||
def pack_graphemes(unpacked)
|
||||
unpacked.flatten.pack('U*')
|
||||
end
|
||||
|
||||
# Re-order codepoints so the string becomes canonical.
|
||||
|
|
@ -143,7 +130,7 @@ module ActiveSupport
|
|||
end
|
||||
|
||||
# Decompose composed characters to the decomposed form.
|
||||
def decompose_codepoints(type, codepoints)
|
||||
def decompose(type, codepoints)
|
||||
codepoints.inject([]) do |decomposed, cp|
|
||||
# if it's a hangul syllable starter character
|
||||
if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
|
||||
|
|
@ -156,7 +143,7 @@ module ActiveSupport
|
|||
decomposed.concat ncp
|
||||
# if the codepoint is decomposable in with the current decomposition type
|
||||
elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability)
|
||||
decomposed.concat decompose_codepoints(type, ncp.dup)
|
||||
decomposed.concat decompose(type, ncp.dup)
|
||||
else
|
||||
decomposed << cp
|
||||
end
|
||||
|
|
@ -164,7 +151,7 @@ module ActiveSupport
|
|||
end
|
||||
|
||||
# Compose decomposed characters to the composed form.
|
||||
def compose_codepoints(codepoints)
|
||||
def compose(codepoints)
|
||||
pos = 0
|
||||
eoa = codepoints.length - 1
|
||||
starter_pos = 0
|
||||
|
|
@ -283,30 +270,27 @@ module ActiveSupport
|
|||
def normalize(string, form=nil)
|
||||
form ||= @default_normalization_form
|
||||
# See http://www.unicode.org/reports/tr15, Table 1
|
||||
codepoints = u_unpack(string)
|
||||
codepoints = string.codepoints.to_a
|
||||
case form
|
||||
when :d
|
||||
reorder_characters(decompose_codepoints(:canonical, codepoints))
|
||||
reorder_characters(decompose(:canonical, codepoints))
|
||||
when :c
|
||||
compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints)))
|
||||
compose(reorder_characters(decompose(:canonical, codepoints)))
|
||||
when :kd
|
||||
reorder_characters(decompose_codepoints(:compatability, codepoints))
|
||||
reorder_characters(decompose(:compatability, codepoints))
|
||||
when :kc
|
||||
compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints)))
|
||||
compose(reorder_characters(decompose(:compatability, codepoints)))
|
||||
else
|
||||
raise ArgumentError, "#{form} is not a valid normalization variant", caller
|
||||
end.pack('U*')
|
||||
end
|
||||
|
||||
def apply_mapping(string, mapping) #:nodoc:
|
||||
u_unpack(string).map do |codepoint|
|
||||
cp = database.codepoints[codepoint]
|
||||
if cp and (ncp = cp.send(mapping)) and ncp > 0
|
||||
ncp
|
||||
else
|
||||
codepoint
|
||||
end
|
||||
end.pack('U*')
|
||||
def downcase(string)
|
||||
apply_mapping string, :lowercase_mapping
|
||||
end
|
||||
|
||||
def upcase(string)
|
||||
apply_mapping string, :uppercase_mapping
|
||||
end
|
||||
|
||||
# Holds data about a codepoint in the Unicode database
|
||||
|
|
@ -374,6 +358,17 @@ module ActiveSupport
|
|||
|
||||
private
|
||||
|
||||
def apply_mapping(string, mapping) #:nodoc:
|
||||
string.each_codepoint.map do |codepoint|
|
||||
cp = database.codepoints[codepoint]
|
||||
if cp and (ncp = cp.send(mapping)) and ncp > 0
|
||||
ncp
|
||||
else
|
||||
codepoint
|
||||
end
|
||||
end.pack('U*')
|
||||
end
|
||||
|
||||
def tidy_byte(byte)
|
||||
if byte < 160
|
||||
[database.cp1252[byte] || byte].pack("U").unpack("C*")
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
# encoding: utf-8
|
||||
|
||||
module ActiveSupport #:nodoc:
|
||||
module Multibyte #:nodoc:
|
||||
# Returns a regular expression that matches valid characters in the current encoding
|
||||
def self.valid_character
|
||||
VALID_CHARACTER[Encoding.default_external.to_s]
|
||||
end
|
||||
|
||||
# Verifies the encoding of a string
|
||||
def self.verify(string)
|
||||
string.valid_encoding?
|
||||
end
|
||||
|
||||
# Verifies the encoding of the string and raises an exception when it's not valid
|
||||
def self.verify!(string)
|
||||
raise EncodingError.new("Found characters with invalid encoding") unless verify(string)
|
||||
end
|
||||
|
||||
# Removes all invalid characters from the string.
|
||||
#
|
||||
# Note: this method is a no-op in Ruby 1.9
|
||||
def self.clean(string)
|
||||
string
|
||||
end
|
||||
end
|
||||
end
|
||||
Binary file not shown.
|
|
@ -7,6 +7,7 @@ class String
|
|||
def __method_for_multibyte_testing_with_integer_result; 1; end
|
||||
def __method_for_multibyte_testing; 'result'; end
|
||||
def __method_for_multibyte_testing!; 'result'; end
|
||||
def __method_for_multibyte_testing_that_returns_nil!; end
|
||||
end
|
||||
|
||||
class MultibyteCharsTest < Test::Unit::TestCase
|
||||
|
|
@ -36,11 +37,15 @@ class MultibyteCharsTest < Test::Unit::TestCase
|
|||
assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id
|
||||
end
|
||||
|
||||
def test_forwarded_bang_method_calls_should_return_the_original_chars_instance
|
||||
def test_forwarded_bang_method_calls_should_return_the_original_chars_instance_when_result_is_not_nil
|
||||
assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing!
|
||||
assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id
|
||||
end
|
||||
|
||||
def test_forwarded_bang_method_calls_should_return_nil_when_result_is_nil
|
||||
assert_nil @chars.__method_for_multibyte_testing_that_returns_nil!
|
||||
end
|
||||
|
||||
def test_methods_are_forwarded_to_wrapped_string_for_byte_strings
|
||||
assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class
|
||||
end
|
||||
|
|
@ -67,17 +72,6 @@ class MultibyteCharsTest < Test::Unit::TestCase
|
|||
assert !@proxy_class.consumes?(BYTE_STRING)
|
||||
end
|
||||
|
||||
def test_unpack_utf8_strings
|
||||
assert_equal 4, ActiveSupport::Multibyte::Unicode.u_unpack(UNICODE_STRING).length
|
||||
assert_equal 5, ActiveSupport::Multibyte::Unicode.u_unpack(ASCII_STRING).length
|
||||
end
|
||||
|
||||
def test_unpack_raises_encoding_error_on_broken_strings
|
||||
assert_raise(ActiveSupport::Multibyte::EncodingError) do
|
||||
ActiveSupport::Multibyte::Unicode.u_unpack(BYTE_STRING)
|
||||
end
|
||||
end
|
||||
|
||||
def test_concatenation_should_return_a_proxy_class_instance
|
||||
assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class
|
||||
assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class
|
||||
|
|
@ -112,15 +106,11 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
|
|||
end
|
||||
end
|
||||
|
||||
def test_indexed_insert_accepts_fixnums
|
||||
@chars[2] = 32
|
||||
assert_equal 'こに わ', @chars
|
||||
end
|
||||
|
||||
%w{capitalize downcase lstrip reverse rstrip strip upcase}.each do |method|
|
||||
%w{capitalize downcase lstrip reverse rstrip upcase}.each do |method|
|
||||
class_eval(<<-EOTESTS)
|
||||
def test_#{method}_bang_should_return_self
|
||||
assert_equal @chars.object_id, @chars.send("#{method}!").object_id
|
||||
def test_#{method}_bang_should_return_self_when_modifying_wrapped_string
|
||||
chars = ' él piDió Un bUen café '
|
||||
assert_equal chars.object_id, chars.send("#{method}!").object_id
|
||||
end
|
||||
|
||||
def test_#{method}_bang_should_change_wrapped_string
|
||||
|
|
@ -419,7 +409,7 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase
|
|||
def test_slice_bang_removes_the_slice_from_the_receiver
|
||||
chars = 'úüù'.mb_chars
|
||||
chars.slice!(0,2)
|
||||
assert_equal 'úü', chars
|
||||
assert_equal 'ù', chars
|
||||
end
|
||||
|
||||
def test_slice_should_throw_exceptions_on_invalid_arguments
|
||||
|
|
@ -506,7 +496,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase
|
|||
|
||||
def test_limit_should_work_on_a_multibyte_string
|
||||
example = chars(UNICODE_STRING)
|
||||
bytesize = UNICODE_STRING.respond_to?(:bytesize) ? UNICODE_STRING.bytesize : UNICODE_STRING.size
|
||||
bytesize = UNICODE_STRING.bytesize
|
||||
|
||||
assert_equal UNICODE_STRING, example.limit(bytesize)
|
||||
assert_equal '', example.limit(0)
|
||||
|
|
@ -605,7 +595,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase
|
|||
else
|
||||
str = input
|
||||
end
|
||||
assert_equal expected_length, chars(str).g_length
|
||||
assert_equal expected_length, chars(str).grapheme_length
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require 'abstract_unit'
|
||||
require 'multibyte_test_helpers'
|
||||
|
||||
class MultibyteUtilsTest < ActiveSupport::TestCase
|
||||
include MultibyteTestHelpers
|
||||
|
||||
test "valid_character returns an expression for the current encoding" do
|
||||
with_encoding('None') do
|
||||
assert_nil ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
with_encoding('UTF8') do
|
||||
assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
with_encoding('SJIS') do
|
||||
assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies ASCII strings are properly encoded" do
|
||||
with_encoding('None') do
|
||||
examples.each do |example|
|
||||
assert ActiveSupport::Multibyte.verify(example)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies UTF-8 strings are properly encoded" do
|
||||
with_encoding('UTF8') do
|
||||
assert ActiveSupport::Multibyte.verify(example('valid UTF-8'))
|
||||
assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8'))
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies Shift-JIS strings are properly encoded" do
|
||||
with_encoding('SJIS') do
|
||||
assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS'))
|
||||
assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS'))
|
||||
end
|
||||
end
|
||||
|
||||
test "verify! raises an exception when it finds an invalid character" do
|
||||
with_encoding('UTF8') do
|
||||
assert_raises(ActiveSupport::Multibyte::EncodingError) do
|
||||
ActiveSupport::Multibyte.verify!(example('invalid UTF-8'))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "verify! doesn't raise an exception when the encoding is valid" do
|
||||
with_encoding('UTF8') do
|
||||
assert_nothing_raised do
|
||||
ActiveSupport::Multibyte.verify!(example('valid UTF-8'))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "clean is a no-op" do
|
||||
with_encoding('UTF8') do
|
||||
assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS'))
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
STRINGS = {
|
||||
'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'),
|
||||
'invalid ASCII' => [128].pack('C*'),
|
||||
'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'),
|
||||
'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'),
|
||||
'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'),
|
||||
'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*')
|
||||
}
|
||||
|
||||
def example(key)
|
||||
STRINGS[key].force_encoding(Encoding.default_external)
|
||||
end
|
||||
|
||||
def examples
|
||||
STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) }
|
||||
end
|
||||
|
||||
KCODE_TO_ENCODING = Hash.new(Encoding::BINARY).
|
||||
update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS)
|
||||
|
||||
def with_encoding(enc)
|
||||
before = Encoding.default_external
|
||||
silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] }
|
||||
yield
|
||||
silence_warnings { Encoding.default_external = before }
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue