From 963c36004cfbe6c041460bab9c05718fc3bd7b58 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 10:50:25 -0300 Subject: [PATCH 01/20] Fix incorrect behavior specified in test. This test was actually specifying the opposite of what it should. --- activesupport/test/multibyte_chars_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 20e56e2c81..006e5c5927 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -419,7 +419,7 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase def test_slice_bang_removes_the_slice_from_the_receiver chars = 'úüù'.mb_chars chars.slice!(0,2) - assert_equal 'úü', chars + assert_equal 'ù', chars end def test_slice_should_throw_exceptions_on_invalid_arguments From 5fb5ced0cba3306e0f39e45eb299fd0ea32cdd4b Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 11:39:45 -0300 Subject: [PATCH 02/20] Remove overidden slice and slice! methods. --- .../lib/active_support/multibyte/chars.rb | 30 +------------------ 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index dcc176e93f..266cfc2eda 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -150,34 +150,6 @@ module ActiveSupport #:nodoc: chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) end - # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that - # character. - # - # Example: - # 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち" - def slice(*args) - if args.size > 2 - raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native - elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp))) - raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native - elsif (args.size == 2 && !args[1].is_a?(Numeric)) - raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native - elsif args[0].kind_of? Range - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps.nil? ? nil : cps.pack('U*') - elsif args[0].kind_of? Regexp - result = @wrapped_string.slice(*args) - elsif args.size == 1 && args[0].kind_of?(Numeric) - character = Unicode.u_unpack(@wrapped_string)[args[0]] - result = character && [character].pack('U') - else - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps && cps.pack('U*') - end - result && chars(result) - end - alias_method :[], :slice - # Limit the byte size of the string to a number of bytes without breaking characters. Usable # when the storage for a string is limited for some reason. # @@ -265,7 +237,7 @@ module ActiveSupport #:nodoc: chars(Unicode.tidy_bytes(@wrapped_string, force)) end - %w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method| + %w(capitalize downcase lstrip reverse rstrip strip tidy_bytes upcase).each do |method| # Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will # exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings. if public_method_defined?(method) From 4387f9724273ee477ff60876b360d0abcee0e344 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 11:40:01 -0300 Subject: [PATCH 03/20] Override #slice! to ensure proper return value. The default pass-through to `method_missing` makes `#slice!` return `self` rather than the string that was sliced off. --- activesupport/lib/active_support/multibyte/chars.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 266cfc2eda..9ca977012c 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -142,6 +142,10 @@ module ActiveSupport #:nodoc: end end + def slice!(*args) + chars(@wrapped_string.slice!(*args)) + end + # Reverses all characters in the string. # # Example: From c4b522d3c89208b554780e9c49747d07dbe3a4e5 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 11:51:10 -0300 Subject: [PATCH 04/20] Make return value from bang methods match Ruby docs The docs for the String class indicate that methods like `rstrip!` and others should return nil when they do not have an effect on the string. --- .../lib/active_support/multibyte/chars.rb | 4 ++-- activesupport/test/multibyte_chars_test.rb | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 9ca977012c..027aadb029 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -47,8 +47,8 @@ module ActiveSupport #:nodoc: # Forward all undefined methods to the wrapped string. def method_missing(method, *args, &block) if method.to_s =~ /!$/ - @wrapped_string.__send__(method, *args, &block) - self + result = @wrapped_string.__send__(method, *args, &block) + self if result else result = @wrapped_string.__send__(method, *args, &block) result.kind_of?(String) ? chars(result) : result diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 006e5c5927..41248cccb6 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -7,6 +7,7 @@ class String def __method_for_multibyte_testing_with_integer_result; 1; end def __method_for_multibyte_testing; 'result'; end def __method_for_multibyte_testing!; 'result'; end + def __method_for_multibyte_testing_that_returns_nil!; end end class MultibyteCharsTest < Test::Unit::TestCase @@ -36,11 +37,15 @@ class MultibyteCharsTest < Test::Unit::TestCase assert_not_equal @chars.object_id, @chars.__method_for_multibyte_testing.object_id end - def test_forwarded_bang_method_calls_should_return_the_original_chars_instance + def test_forwarded_bang_method_calls_should_return_the_original_chars_instance_when_result_is_not_nil assert_kind_of @proxy_class, @chars.__method_for_multibyte_testing! assert_equal @chars.object_id, @chars.__method_for_multibyte_testing!.object_id end + def test_forwarded_bang_method_calls_should_return_nil_when_result_is_nil + assert_nil @chars.__method_for_multibyte_testing_that_returns_nil! + end + def test_methods_are_forwarded_to_wrapped_string_for_byte_strings assert_equal BYTE_STRING.class, BYTE_STRING.mb_chars.class end @@ -117,10 +122,11 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase assert_equal 'こに わ', @chars end - %w{capitalize downcase lstrip reverse rstrip strip upcase}.each do |method| + %w{capitalize downcase lstrip reverse rstrip upcase}.each do |method| class_eval(<<-EOTESTS) - def test_#{method}_bang_should_return_self - assert_equal @chars.object_id, @chars.send("#{method}!").object_id + def test_#{method}_bang_should_return_self_when_modifying_wrapped_string + chars = ' él piDió Un bUen café ' + assert_equal chars.object_id, chars.send("#{method}!").object_id end def test_#{method}_bang_should_change_wrapped_string From 7301aa2e0d3b16b96fcbffcd79ee9bbf6bc65c57 Mon Sep 17 00:00:00 2001 From: Sergey Nartimov Date: Thu, 5 Jan 2012 18:25:27 +0300 Subject: [PATCH 05/20] refactor AS::Multibyte::Chars --- .../lib/active_support/multibyte/chars.rb | 44 +++---------------- activesupport/test/multibyte_chars_test.rb | 2 +- 2 files changed, 6 insertions(+), 40 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index dcc176e93f..1a9c2c7ef6 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -150,34 +150,6 @@ module ActiveSupport #:nodoc: chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) end - # Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that - # character. - # - # Example: - # 'こんにちは'.mb_chars.slice(2..3).to_s # => "にち" - def slice(*args) - if args.size > 2 - raise ArgumentError, "wrong number of arguments (#{args.size} for 1)" # Do as if we were native - elsif (args.size == 2 && !(args.first.is_a?(Numeric) || args.first.is_a?(Regexp))) - raise TypeError, "cannot convert #{args.first.class} into Integer" # Do as if we were native - elsif (args.size == 2 && !args[1].is_a?(Numeric)) - raise TypeError, "cannot convert #{args[1].class} into Integer" # Do as if we were native - elsif args[0].kind_of? Range - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps.nil? ? nil : cps.pack('U*') - elsif args[0].kind_of? Regexp - result = @wrapped_string.slice(*args) - elsif args.size == 1 && args[0].kind_of?(Numeric) - character = Unicode.u_unpack(@wrapped_string)[args[0]] - result = character && [character].pack('U') - else - cps = Unicode.u_unpack(@wrapped_string).slice(*args) - result = cps && cps.pack('U*') - end - result && chars(result) - end - alias_method :[], :slice - # Limit the byte size of the string to a number of bytes without breaking characters. Usable # when the storage for a string is limited for some reason. # @@ -265,14 +237,10 @@ module ActiveSupport #:nodoc: chars(Unicode.tidy_bytes(@wrapped_string, force)) end - %w(capitalize downcase lstrip reverse rstrip slice strip tidy_bytes upcase).each do |method| - # Only define a corresponding bang method for methods defined in the proxy; On 1.9 the proxy will - # exclude lstrip!, rstrip! and strip! because they are already work as expected on multibyte strings. - if public_method_defined?(method) - define_method("#{method}!") do |*args| - @wrapped_string = send(args.nil? ? method : method, *args).to_s - self - end + %w(capitalize downcase reverse slice tidy_bytes upcase).each do |method| + define_method("#{method}!") do |*args| + @wrapped_string = send(method, *args).to_s + self end end @@ -282,10 +250,8 @@ module ActiveSupport #:nodoc: return nil if byte_offset.nil? return 0 if @wrapped_string == '' - @wrapped_string = @wrapped_string.dup.force_encoding(Encoding::ASCII_8BIT) - begin - @wrapped_string[0...byte_offset].unpack('U*').length + @wrapped_string.byteslice(0...byte_offset).unpack('U*').length rescue ArgumentError byte_offset -= 1 retry diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 20e56e2c81..ea02ef0603 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -506,7 +506,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase def test_limit_should_work_on_a_multibyte_string example = chars(UNICODE_STRING) - bytesize = UNICODE_STRING.respond_to?(:bytesize) ? UNICODE_STRING.bytesize : UNICODE_STRING.size + bytesize = UNICODE_STRING.bytesize assert_equal UNICODE_STRING, example.limit(bytesize) assert_equal '', example.limit(0) From 60bbdf7d83a5cc1ee5be053d438fc56172b1ea84 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 15:43:06 -0300 Subject: [PATCH 06/20] Just delegate a few methods directly to @wrapped_string --- .../lib/active_support/multibyte/chars.rb | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index fbc469ae12..90544a2e64 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require 'active_support/core_ext/string/access' require 'active_support/core_ext/string/behavior' +require 'active_support/core_ext/module/delegation' module ActiveSupport #:nodoc: module Multibyte #:nodoc: @@ -38,6 +39,8 @@ module ActiveSupport #:nodoc: alias to_s wrapped_string alias to_str wrapped_string + delegate :<=>, :=~, :acts_like_string?, :to => :wrapped_string + # Creates a new Chars instance by wrapping _string_. def initialize(string) @wrapped_string = string @@ -61,11 +64,6 @@ module ActiveSupport #:nodoc: super || @wrapped_string.respond_to?(method, include_private) end - # Enable more predictable duck-typing on String-like classes. See Object#acts_like?. - def acts_like_string? - true - end - # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise. def self.consumes?(string) # Unpack is a little bit faster than regular expressions. @@ -77,21 +75,6 @@ module ActiveSupport #:nodoc: include Comparable - # Returns -1, 0, or 1, depending on whether the Chars object is to be sorted before, - # equal or after the object on the right side of the operation. It accepts any object - # that implements +to_s+: - # - # 'é'.mb_chars <=> 'ü'.mb_chars # => -1 - # - # See String#<=> for more details. - def <=>(other) - @wrapped_string <=> other.to_s - end - - def =~(other) - @wrapped_string =~ other - end - # Works just like String#split, with the exception that the items in the resulting list are Chars # instances instead of String. This makes chaining methods easier. # From 4b5a3d7367ecdfcff1c2ffe1b299a9865d66ab7e Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:14:39 -0300 Subject: [PATCH 07/20] Remove useless parens --- activesupport/lib/active_support/multibyte/unicode.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 754ca9290b..0197d86adc 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -123,7 +123,7 @@ module ActiveSupport # Example: # Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि' def g_pack(unpacked) - (unpacked.flatten).pack('U*') + unpacked.flatten.pack('U*') end # Re-order codepoints so the string becomes canonical. From d2a4acdbe3027e447c888558bb36d1aef2bfccf9 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:17:12 -0300 Subject: [PATCH 08/20] Update to Unicode 6.0 --- .../lib/active_support/multibyte/unicode.rb | 2 +- .../active_support/values/unicode_tables.dat | Bin 813343 -> 877274 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 0197d86adc..b35e7b9b80 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -10,7 +10,7 @@ module ActiveSupport NORMALIZATION_FORMS = [:c, :kc, :d, :kd] # The Unicode version that is supported by the implementation - UNICODE_VERSION = '5.2.0' + UNICODE_VERSION = '6.0.0' # The default normalization used for operations that require normalization. It can be set to any of the # normalizations in NORMALIZATION_FORMS. diff --git a/activesupport/lib/active_support/values/unicode_tables.dat b/activesupport/lib/active_support/values/unicode_tables.dat index 4fe0268cca05e7912a9fdcb8ce50dbc47b8ff844..7edc4663e86f4835ef82a2f9ed8cb95c1d5b47f5 100644 GIT binary patch delta 35798 zcmZYIcbrVu{|0b(W}BVawYF%h6D*c(SXS?y)mC5KjuKXl7NSJ&VuBET_0H&BlqgX` zh(z@0Wf$Ls#P8hsJkGQG%O76ieV%jX?45Jx-t%!szP}WC<3t%B51)3vqb#%AZ}q9s z%4(UOvb|xeA&GHCJp09Evc-8hGR1kTUp@{?QO~%{4oeZwINyjkTZH-NbP3aaR8A#| zZr`)>V@@Og+{m*bM*i)aFIapmiC%M!ej!HU_BGd5mbX;(aT;}vV0%@cPh(YsRMS!& zQe8_8NDVDDAvLwsg4EJd+sAn^)f!^cPzRAZTIxdTYN-dQr=>omzLo}VNjGpqALo(P zmUPeveH!ULjUkP-G=VhH(iGBEOFSfAOEX9_EzKd#wX}e=(9#mpQcEj!aNeXpI0=X( z=tyfwYb|Yj%=gmXMnhYHwi?X3YZ+ksG~a;&12qg17^Gpaz+eqS1cqoBYJ$Zw)Rn7& zQKPUW%umjdFuzYPQe4C}u%GeRuv9>M1>IihQ+u*;->@{6KN*c(O$HjXql{_J_<2T% zZCaL3!%Y{Mu3?723=K2gV9ZRpw#vwCT$kqSy7`7Q-*BDZWX$Yo=@2wG`C2wx(5LY` z2F-odC&b=SL!uj8gR)t?9vO$7)gG&J<#6by_z{648jcDa)o@JUn1L`ga~jSIoY!zc;DUyq1b)(RQQ)G6 zO9GcPTo$;j;flZ&4L=L~tl_G_RSnk!u4(v1;1>cMZ1$ZfW>K;13PA1#WA&BXCEjUAv7MPKZYM% z&&ycnbvM3oR-11P$^K3LINy2&eFhHh;uU9a5oc@X<>>3M%$LUb|84B=usEgB`QOH5 zhvi#o{Pw?%100qG(zxKijRPH)h0?h2zm0<(mPOLI=)a9$I4s|}8-u_5Z{sl6m_^oT ztFgzOF!s3CEwN5;7?0*&w5rR*qe7oP;~on<*6>8&iH4^FPc=Lfc&6dGz;g{R1YT%( zDezLmD}h%UUJJa|kSdU>A(PVx^ayd)^6^;E!E(NcT#C*Y`SgQa5OJk-_o#2N+(G*t z-F_ExSIa%fJuUZ(IOC`KgxEAZDB@gJ%`+W5M4yK`@~4PvP?AS4tKm7fnuj{Ro<%=R z%gY33YHyF;hOe~w>NcBzO@rM8*T6m=y{$$uX)LB2!`zLj{X80Y7!9Sdp>Av>&`3jL zfyNq|nBW{ZNKK@A!$6M8xOl1n}4N#NaMUT zp4W{R1TJX!$(_DD*5lX3mj289!gGjop!- zdh8L{qhYTbjJ*>hHhQ`uHhMj+9M@@ex*dvp57T+IxkKr+`qL<#(qJWWghe^VP&tgTCVj#^k+tqI)<~kL+K{eIAF-$g45ES< z5AUAsXvHB`gq&7AwT6#RKA@#S`MG#cRFum5)b=O+#ixGHdaL z_-e61*vk&F>x>`7Pm4dqUrSa$KrKO#AT7a=U@al*ofm!6{}9;_$)+RuA^Eiw zfE3VD5K>S}AxI%Dg&~Ete5PL9p8my)s6#ryJKZ5IhCan~pD;+6mg11&T1r4lXekLP zsU>{9^Y(JJJ<(7~B9>C?U19CL-y|E`oUgX3xoms?Y2j=a*sfuRzzz*N1$JuKC9q3_ zOTeXJx4>=_=xyW zGK*xU_=@;aY$7&_UBpiD6Y-AaisYo^63Img6$z!}7RgP?Ba(-bS0pbbpGZDRev$l?0wM({1w{%{ z3W*e=6c#B=`Ap<9N)eGFl%gU+K9BFv=wPfX(!T-(q5!JrGrQZN=K27ltht4N+*#{l+GfZDP2UmP`ZkArF0YN zM(Hlnozg?32c@S-PfC(V62&3np!5>yMd>Zlo6<+552de2U&`kqpHupY^rMV%%fz2! zGP{0`@g0`aNOCTl?GxgcDT!BVkU_G5EF>$)O7Z}CkUT-2BrlK`$s6QN@=0XZ(WwLoGrEQcZp6=A3-s3@rzs2C{>6h&YK_jzFm@b5aST-iRU}mcRU%aeRVGyd zRUuVXFL)}$M5`fGjiKtG>ZBT=8l;+_nxtBwTBO>b+N3(5I;6Uwx}|@ z6wfnEy)#0c8R`P+Lh1_YO6mscM(UpAtadxY&)fr{9t`ya^&};Ml1L7agVamSeOHFV z&>NxN4D|u^A@v3IC4H_Y`gw-K&<~-04D|=~CnbZDNdrIwNCQCwNrOOxNP|IxNkc$G zNJBwGNnfb5;-?IgABNB{hK7TNlSY6>kVb+=l170>k-h|dNg53rO&S9lL;5PonVgtm zJ&Z+YEJNc!<4EH{<4F@h6G&gH(dTA34igcY$j~IvB+_KiWYQGS6w*}CRMIriG}3g? zbkd9@=T}WLOlu}WGZ~r%nnjuonoXJmnnRkaXl#Z%Sg*X%SkIh zD@ZFrD@m(Bt4OQWoxJ4id#p~;(JZS7JEerbl!Wf!`+*ljmxH))UB zI`+u0l=mXEm!W;2eWd-M{iFk+1EhnXgQP>CL!`r?!=xjiBc!9CqoiY?W2EDt7N?pdU$RL1#(lK<7y3LFY*qKo>|qsW<b%*$jv8F}giww#q-UUKq~~hAr(`(DFA#dc z&`Z!u(kr#O{yf91UL*9Hp;S;RDGiiHdINexdJB3>dIx$(`XA_jq`yFak^Tn#O?nS{ zPx=7*K>7#t59wdEZ%)kc?tDb(BSV=SC}}xZ(gGPI3&=vUf~+JDkO#>V>xYI59CMk2l9-0uc%%1%ZM{ z!JuGL2q=VL@Ep_O!`bM5VNF3DS}WDQc+M*QZZ05QWz+VR2)>CR033j zR1#E@6b=d}l>(I_m3Bx;ixPoQ1St{}Ns0nRk)lD-q!>^PsfJLe)ssLDfk$ zKs87;K{ZLWK($D98%Jv)JLd3sR5_~sUfH#sS&6Vsj(W}ENM}i zAk>7^6x5Uy4~i!>12rQx2Q??P0JR{s1hpi!a!5&wl7LVGskK^=W=V_E2B9{jwxG78 zcA$2o_UiS_k`|={LLEpQK^;knphQw9byAomElOvEI+MD9x{$hpx{|tqx{Xoq@kdpq%YK2VV1Ng!w?!q8V(vx8UY$X8VMRn8U-3f z`V#adX*6gwX$)u#=_`knv?yZ{8cP}n8b=xr8c&)4Qjb!9dWPt0HM&{SqD(|+B54w6 z5@|AMGHD8E3TY~6Drp*M8fiLcI%$SON?Mee2+btT0?i`L2F)hT0nH)JRb-a5DDx1S zNBTxxKg^OAWj;dlNluWH^sPEA%#s#m0YVE%3qcD>i$IG=-+{g(Ee0(nEdebdeGmGc zv=p?Iv<$S2v>ddYv;wq(v=X$Evm^`P~n z4WJFAji8OBO`uJr&7jSsE$X~5OInnz2yG>818pO12W=W2llG`*AIy>#WiLW|N&7(iNc%ziNe4g&NC!a&Nrym(NQXg(Nk>3ONJl|O zNyk9PNXJ3PNhd%jNIy8Fq(wQ2&`HuM&?(Ys&}q^c&>7N?pdU$RL1#(lK<7y3LFY*q zKo>|qsW;y&X;CgBbdhukbcu8sbeVJobcOV@y1<(yEy`7du9B{Su91FG_eW+)i*g;I z>!cf?8>C;=$DLWyqTEF2Ch0fOZ=~PV&de-nQEnl0i}VNR57KSWZPFdk9nxLUUD7?! zJ<@&9ebNKa1JXm#L(-q>tTRhmlt=1KHA`BQ$LRK$^aS*T^c3`z^bGWj^jxiXv!q3N zfzS)mOVCTwE48^cOInoI2)!nyf>KFopfu7O&>PZQ&|A_w&^yxqK>s8C1^SEhH|THD zd(eB*2ha!7KcIg||EhhnS<<3>MCc%s zMe+uDlYDwfNsE%Xmz1<9zUbylvVm+QJIGG*1No8sLH?vHpe&>SPyi`wFDYqJ0uc%% z1%ZM{!JuGL2q=VDC0I49TAgK_j5UDVzFzGY3K+KXBr3gYrNJT+KNyR|LNMWEbQgKjm zQVCEAQb|xrQaC7_R0>pzRJxaxv?vh>MUWyvk)$Y46e$`MO^N}=)TjIUsU? za!}+T<%q};%2AP{lw%^tD91&PQ%;DSp!^{61LdU1Ny;gaQ&Sei8YFa$V#) z<%Y-&%C927Qf`Xer2HoG8|8PA-zm4;V%&1I?QdJ+V?;P3lWp15Z`Fp`vU${YSR>uw z%reZD%ZQT3C~2(gutp0+3)FL1V+3LZ>N~7u1j-0Ba9GO`u$B`jC(y`YjTML$ zXzZ|-7bq{##9^%uBsi?qCB58C*O0~<(%8mfttn7bpsmANOQ4oOJBPKlbiZtNuOp3h zq_Km;T34X1Ku3qQo9cdPh=kD8m7nGMGFDb7?UQu3)yr!gzq*BsE(kO34-ca6(yrsMoc}Mx5$p0vRiTp+R zTjX!bdy)5)4iS$kMEzTFYPyzPEnWS` z*+MKvQ#V_izBZi6Cp|+vo$3l0FBT8;dePir&6J7_QL4V`86bmX0a-{^kd@>C@*sJF zJV{<4FOoONo8*%!1r$nVwNA~ixlw%4&6i{Y*+_O!x{iJzKc@Ua{-i9RETjNX04Zy# z6i_IE2nCXYKtZHnP%tS36hg|TMmGy6lIn5~(t%GN}rv3aP4kLGv*eN;QP4k*b5LlWKr!kZOWz zl4^l!k!pi#lj?x#km`c!lIo?(V=k2X2-PPw05u>r1T`cz0yQEvR->Dbxlo!Q)P&R& z)RYttiYGM#H6t|#H7B(IwIHW2llG*_vmBJY2<;{9 z1MMU22kj>v039G51RW$D0v#e91|23H0UaS71sx?F105qB2OTHjWr(^pgdBG z#(Z3W@)+G7lb(Q{ke-5`lAeK{k)ErgY(6ePd4bRi(o4`wQl>O)>C(8R0~sU>$U?G$ ztRxSR2gwuUN%8`Dk-S0PB%d_d(or(2LW8-bqxhnmFUbb7k?f#!9sNLlO!1eGL(gThIrK&42f)8y~ODG>-okRm~mq$p4nDH;?_ ziUGxt%B0C|mr^!O{!W}yE=~5olvoTDODYd4PpY8SsJZ{8#32+%stBq`ssyS;stl@3 zs;WjeH@lST2vsN50M#JX1l1(f0@Whb2Gu6j0o5VZ1=S_hOOu@~r9MLSNew^^NDV;^ zNsT~_NR8F#=FXPV1feFRrl6*zcu+j48K@blIjA|Q1*iq7C8#B-RhsN5!8{C2udV%Qp?@^0tcltLY+xnKwU^( zL0w7RK;1~)(`08$>48uWQcqA%QW7YMyNMj8$p zP8tCkK^h4fNg4$jMfwu-C22HhG-(WI4C$*h*%DL6A~cpX4m6H59yFdb0W^X1wHn>r z5>qB3G?6q3G>J4BG?_F7G=(%3G?g?BG>tSJG@UdfO}508nF!4!%>vCL%?8aT%>m6J z%~fP>i7E3Cnn(IZT`bHma8TwWG@s-IIZ5BD)581$2W0_53rGt=3rUMWi%8#rz9TIL zEha4iEg^jm`ku5Dw3M_Aw2ZVIw4Ag8w1Tt}w34(6w2HJ^-2<4Leo6{LDWo-^HKetm zwWM{Rb)@y6^`s4;4Wx~rjigPWO{C4B&7>{ryf8QYl&uJDC2a$3BW(w5C+z_3AnjC# z#@zH%cBut#Zu%)MbaRn*gLaelq{*hAvKOJfqq=TS?q(h)Xq{E=Y zq$8jsq@$ptq+_6Cq~oCDq!XYMq#x3x4nR4H&`HuM&?(Ys&}q^c&>7N?pdU$RL1#(l zK<7y3LFY*qKo>|qsW;#J0te+HLKjJwK$l3DL6=EaKvzgVs|&pO1rEwpgszgVfv%B$ zQTIpY7dR-_5xP#g0lGo@Rejuet-3Q$#Jper*Jp?@@{i)76vm!%zq~27sB13tMZjVV%Ku<_dK~G7~ zK+j0e)p|E8GL#nxy&$~=y(GO-+h+3%9F*4xy(XoCQb}o`G}0T;8`4|QThcqwJJMgE zzesy;Y-)6~!bHiAPN-72_MhXLkk&1(glS+U}kV=9| zlEOjZq*9<#q|$Gs!bFKcD1sCTiX=sWqDax8Xi^L)hE(Q_RG27b-$;dtQtpjZm?*Ir zD3(+nRGw5ptx>bWM2SNvj#Lp;kyHs(iPRL-loSt&Cp7~#BQ*y#C$#{zAhiUwB(-`Y zMH5N_LJ6eS>gbt86G|I|+K}3U+LGFV+L79;*E5SIlnw}WAaw+FBqf3pNuAU?XckQ< zoe}Cx>H_LQ>I&*g>IUjY>i$NGCX^ls^&s^G^(4Iqy(fJDeIWe<`iJzdIw#C82~s{H z^pRw|mCKd!R$s0xAPdO~vXVSN9wbkYC&>%sMe+uDlYHLFwTY5hJwj<-n<&2M=1a1H zY$Q7CJR=C^sn& zC=V$wC@(4hTe((H3LsQ~R1j2zJO8-p<1NcpxUH5pgN?wpt_`bZ>8){sgF>7 zQUg!}QbSNfQX^0!Qe!o`S@x$iL8u973}_7LtG7~aq>M#qENL8Q9BDjgJZS=G0_kft zx>;_dOhjlRX%c7>X)bGFG@CRB zG>0@-ky&n}%tL4%=^M4y%yJ`TK0@_khBQ2i1Z!kJJMp% zV$u@O64Lje?@3EROG(Q>%Sg*X%SkIhD@ZFrD@m(Bt4OQWr;1r_q@*B}LRte_Ls|=3 zOIin7M_LbBPuc+5K-vh}NZJJ2MA{76OxmK(3$xrv*^1Cs(l*dG(ss~x(hkrL(oS_~ z%yJ`Tms;>RHy3F)Xg6ukTPZhE_9C>Gv=6k8v>&vebO3aKbP#lqbO>~abQpA) zbOdySbQE-ybPRNibR2Y?bOLmOl<6JH4)64RoB=XO7LbKx1zAZRAPa5gZxQZKv_rupa4?VcT#qs1R@kj z3IYX@fN-72_MhXLkk&1(g zlS+U}kV=9|lEOjZq*9<#q|)!C>_CY?D1sCTiX=sWqDax8Xi^L)hE(RA+`CfBzLR@b zO1XD(?@EcqK(VCqpz@>&YK@xru9P^0;z$)i6-kvql}MFAl}S}VRY+CU3!3+?lxhf7 zBUJ}gC)EJeAk_rbB-H}dBGm@fCe;DeA=L%dCDnT;_ra9<2-PPw05u>r1T`cz0yQEv zR->Eu!IUNlH6b+xH6_J^;z`Xw%}C8b%}FglEl4dvElI83NhODpfKURdwOWv7C5O@m zp*EzppthuTpmwD8>h;V@4y6M^9Y`HP9Z89xL{cYpQkaz-N@s*Rle&Pqkh+4plDdJq zk-EQ=N)Dw5LOn=5K|M)Hpd^w5nS(52-JxFX?kNQS@KL8a_VawK{=Ed9Ou#Rd4pWvv|-=&)S%iVJdBZaUAEvY+>E z<757zopWYsdp4tjb9rfdxRK!cskFU|B{e}vX^{FC8UkELW9`)ppCJy5RsFBs_r0^v^36azFZSm zWCeS;C9lI0k^X0ia{nyR{tinm^=pfBWRyLh%TdvO$DH-U^wWFnp59Y6>*wkDwVP92 Y^())M44+H^hWa*dfZQcm!OZsIRGG@f zmTHn))FbCOe0v@Ky$;8|9ldhjk5Gv6d55&#jG+mscjb%oS$I%`VhMx7!AZC`0v=#% zBwQ8X5D$t14Zm68Ts$=J@m#&wwGFJ+&dy38_^=A50pA>g#x(rU4L@a!>VxBZ@O?N5 ze~)5-P`w;4ZxBxIh7Rs+6EwWiC4>n$c3n8?h6Vhqm1;N~O3#*%r!m|W2Ell#iWU$u z5yLC2vZln?Eh%;-#ZBJ*Yu`Ui%LEX8>u4}{*3fcL=MW=nX}JJ#2KP76O%%4wu}Zes z{-3bb$kciCjKi37Hc|z^WoAy$auLoL+|f*@2=tiVN4o&}P2Z|VVaQ-7r>`!MxN^v8Oqr8(3yIcUMjK_w&_M?NbpGNkhQ?@9>2@k?0@t4*H%t@B!wOQv;RZ}OKp z&NKP;jItPAv-$)1u|G<5t51#Qc0rAXBWAx%P!qXCbV#3Q+1SM;+eJ4yAnvyZ z%J!f<`$9K)CfXOd$*8Vh^G7!Jgm=d>3%?V~mO&A|jb%GYK}vv|qz25(qy@}n(&wuT zL#@&$bLZ=ghA!0gO?~z7$C)gG*Tgd~H1og&Hh|*^tQ+{ Date: Thu, 5 Jan 2012 16:21:59 -0300 Subject: [PATCH 09/20] Assume Encoding support --- activesupport/lib/active_support/multibyte/chars.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 90544a2e64..1505d3e604 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -66,11 +66,7 @@ module ActiveSupport #:nodoc: # Returns +true+ when the proxy class can handle the string. Returns +false+ otherwise. def self.consumes?(string) - # Unpack is a little bit faster than regular expressions. - string.unpack('U*') - true - rescue ArgumentError - false + string.encoding == Encoding::UTF_8 end include Comparable From b81bef531c9ebe08d39a4859b01824b390866675 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:26:24 -0300 Subject: [PATCH 10/20] Just use Ruby's String#[]= --- .../lib/active_support/multibyte/chars.rb | 41 ------------------- activesupport/test/multibyte_chars_test.rb | 5 --- 2 files changed, 46 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 1505d3e604..8cd3cfe455 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -80,47 +80,6 @@ module ActiveSupport #:nodoc: @wrapped_string.split(*args).map { |i| i.mb_chars } end - # Like String#[]=, except instead of byte offsets you specify character offsets. - # - # Example: - # - # s = "Müller" - # s.mb_chars[2] = "e" # Replace character with offset 2 - # s - # # => "Müeler" - # - # s = "Müller" - # s.mb_chars[1, 2] = "ö" # Replace 2 characters at character offset 1 - # s - # # => "Möler" - def []=(*args) - replace_by = args.pop - # Indexed replace with regular expressions already works - if args.first.is_a?(Regexp) - @wrapped_string[*args] = replace_by - else - result = Unicode.u_unpack(@wrapped_string) - case args.first - when Fixnum - raise IndexError, "index #{args[0]} out of string" if args[0] >= result.length - min = args[0] - max = args[1].nil? ? min : (min + args[1] - 1) - range = Range.new(min, max) - replace_by = [replace_by].pack('U') if replace_by.is_a?(Fixnum) - when Range - raise RangeError, "#{args[0]} out of range" if args[0].min >= result.length - range = args[0] - else - needle = args[0].to_s - min = index(needle) - max = min + Unicode.u_unpack(needle).length - 1 - range = Range.new(min, max) - end - result[range] = Unicode.u_unpack(replace_by) - @wrapped_string.replace(result.pack('U*')) - end - end - def slice!(*args) chars(@wrapped_string.slice!(*args)) end diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index f25e062873..0c6b03f15f 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -117,11 +117,6 @@ class MultibyteCharsUTF8BehaviourTest < Test::Unit::TestCase end end - def test_indexed_insert_accepts_fixnums - @chars[2] = 32 - assert_equal 'こに わ', @chars - end - %w{capitalize downcase lstrip reverse rstrip upcase}.each do |method| class_eval(<<-EOTESTS) def test_#{method}_bang_should_return_self_when_modifying_wrapped_string From 262af66d0578b748fb878e2b0d36eb40af722efc Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:36:09 -0300 Subject: [PATCH 11/20] Document method definition --- activesupport/lib/active_support/multibyte/chars.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 8cd3cfe455..ba2ada2c1a 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -80,6 +80,8 @@ module ActiveSupport #:nodoc: @wrapped_string.split(*args).map { |i| i.mb_chars } end + # Works like like String#slice!, but returns an instance of Chars, or nil if the string was not + # modified. def slice!(*args) chars(@wrapped_string.slice!(*args)) end From c973161c388dbf4e175c59fd8939b794c5f662cb Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:36:21 -0300 Subject: [PATCH 12/20] Remove unused code. --- .../lib/active_support/multibyte/chars.rb | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index ba2ada2c1a..fc0650dbbe 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -202,31 +202,6 @@ module ActiveSupport #:nodoc: end end - def justify(integer, way, padstr=' ') #:nodoc: - raise ArgumentError, "zero width padding" if padstr.length == 0 - padsize = integer - size - padsize = padsize > 0 ? padsize : 0 - case way - when :right - result = @wrapped_string.dup.insert(0, padding(padsize, padstr)) - when :left - result = @wrapped_string.dup.insert(-1, padding(padsize, padstr)) - when :center - lpad = padding((padsize / 2.0).floor, padstr) - rpad = padding((padsize / 2.0).ceil, padstr) - result = @wrapped_string.dup.insert(0, lpad).insert(-1, rpad) - end - chars(result) - end - - def padding(padsize, padstr=' ') #:nodoc: - if padsize != 0 - chars(padstr * ((padsize / Unicode.u_unpack(padstr).size) + 1)).slice(0, padsize) - else - '' - end - end - def chars(string) #:nodoc: self.class.new(string) end From 9ea34ad3875243b9dd57a7fbd9ff6fa9c55872ac Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 16:43:28 -0300 Subject: [PATCH 13/20] Remove "_codepoints" from compose/decompose --- .../lib/active_support/multibyte/chars.rb | 4 ++-- .../lib/active_support/multibyte/unicode.rb | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index fc0650dbbe..1389cb50c9 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -153,7 +153,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 2 # 'é'.mb_chars.decompose.to_s.length # => 3 def decompose - chars(Unicode.decompose_codepoints(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.decompose(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) end # Performs composition on all the characters. @@ -162,7 +162,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 3 # 'é'.mb_chars.compose.to_s.length # => 2 def compose - chars(Unicode.compose_codepoints(Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.compose(Unicode.u_unpack(@wrapped_string)).pack('U*')) end # Returns the number of grapheme clusters in the string. diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index b35e7b9b80..c689c14631 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -143,7 +143,7 @@ module ActiveSupport end # Decompose composed characters to the decomposed form. - def decompose_codepoints(type, codepoints) + def decompose(type, codepoints) codepoints.inject([]) do |decomposed, cp| # if it's a hangul syllable starter character if HANGUL_SBASE <= cp and cp < HANGUL_SLAST @@ -156,7 +156,7 @@ module ActiveSupport decomposed.concat ncp # if the codepoint is decomposable in with the current decomposition type elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability) - decomposed.concat decompose_codepoints(type, ncp.dup) + decomposed.concat decompose(type, ncp.dup) else decomposed << cp end @@ -164,7 +164,7 @@ module ActiveSupport end # Compose decomposed characters to the composed form. - def compose_codepoints(codepoints) + def compose(codepoints) pos = 0 eoa = codepoints.length - 1 starter_pos = 0 @@ -286,13 +286,13 @@ module ActiveSupport codepoints = u_unpack(string) case form when :d - reorder_characters(decompose_codepoints(:canonical, codepoints)) + reorder_characters(decompose(:canonical, codepoints)) when :c - compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints))) + compose(reorder_characters(decompose(:canonical, codepoints))) when :kd - reorder_characters(decompose_codepoints(:compatability, codepoints)) + reorder_characters(decompose(:compatability, codepoints)) when :kc - compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints))) + compose(reorder_characters(decompose(:compatability, codepoints))) else raise ArgumentError, "#{form} is not a valid normalization variant", caller end.pack('U*') From 51648a6fee31c9642d3ce8899a1c718e1604f4bc Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:02:14 -0300 Subject: [PATCH 14/20] Remove multibyte utils This is neither a public API, nor used internally, so let's remove it. --- activesupport/lib/active_support/multibyte.rb | 21 +---- .../lib/active_support/multibyte/utils.rb | 27 ------ activesupport/test/multibyte_utils_test.rb | 93 ------------------- 3 files changed, 1 insertion(+), 140 deletions(-) delete mode 100644 activesupport/lib/active_support/multibyte/utils.rb delete mode 100644 activesupport/test/multibyte_utils_test.rb diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index 57e8e24bf4..cabe073616 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -21,24 +21,5 @@ module ActiveSupport #:nodoc: def self.proxy_class @proxy_class ||= ActiveSupport::Multibyte::Chars end - - # Regular expressions that describe valid byte sequences for a character - VALID_CHARACTER = { - # Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site) - 'UTF-8' => /\A(?: - [\x00-\x7f] | - [\xc2-\xdf] [\x80-\xbf] | - \xe0 [\xa0-\xbf] [\x80-\xbf] | - [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] | - \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] | - [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] | - \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn, - # Quick check for valid Shift-JIS characters, disregards the odd-even pairing - 'Shift_JIS' => /\A(?: - [\x00-\x7e\xa1-\xdf] | - [\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn - } end -end - -require 'active_support/multibyte/utils' \ No newline at end of file +end \ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/utils.rb b/activesupport/lib/active_support/multibyte/utils.rb deleted file mode 100644 index bd6d4bad41..0000000000 --- a/activesupport/lib/active_support/multibyte/utils.rb +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Returns a regular expression that matches valid characters in the current encoding - def self.valid_character - VALID_CHARACTER[Encoding.default_external.to_s] - end - - # Verifies the encoding of a string - def self.verify(string) - string.valid_encoding? - end - - # Verifies the encoding of the string and raises an exception when it's not valid - def self.verify!(string) - raise EncodingError.new("Found characters with invalid encoding") unless verify(string) - end - - # Removes all invalid characters from the string. - # - # Note: this method is a no-op in Ruby 1.9 - def self.clean(string) - string - end - end -end diff --git a/activesupport/test/multibyte_utils_test.rb b/activesupport/test/multibyte_utils_test.rb deleted file mode 100644 index f807492be0..0000000000 --- a/activesupport/test/multibyte_utils_test.rb +++ /dev/null @@ -1,93 +0,0 @@ -# encoding: utf-8 - -require 'abstract_unit' -require 'multibyte_test_helpers' - -class MultibyteUtilsTest < ActiveSupport::TestCase - include MultibyteTestHelpers - - test "valid_character returns an expression for the current encoding" do - with_encoding('None') do - assert_nil ActiveSupport::Multibyte.valid_character - end - with_encoding('UTF8') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character - end - with_encoding('SJIS') do - assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character - end - end - - test "verify verifies ASCII strings are properly encoded" do - with_encoding('None') do - examples.each do |example| - assert ActiveSupport::Multibyte.verify(example) - end - end - end - - test "verify verifies UTF-8 strings are properly encoded" do - with_encoding('UTF8') do - assert ActiveSupport::Multibyte.verify(example('valid UTF-8')) - assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8')) - end - end - - test "verify verifies Shift-JIS strings are properly encoded" do - with_encoding('SJIS') do - assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS')) - assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS')) - end - end - - test "verify! raises an exception when it finds an invalid character" do - with_encoding('UTF8') do - assert_raises(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte.verify!(example('invalid UTF-8')) - end - end - end - - test "verify! doesn't raise an exception when the encoding is valid" do - with_encoding('UTF8') do - assert_nothing_raised do - ActiveSupport::Multibyte.verify!(example('valid UTF-8')) - end - end - end - - test "clean is a no-op" do - with_encoding('UTF8') do - assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS')) - end - end - - private - - STRINGS = { - 'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'), - 'invalid ASCII' => [128].pack('C*'), - 'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'), - 'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'), - 'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'), - 'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*') - } - - def example(key) - STRINGS[key].force_encoding(Encoding.default_external) - end - - def examples - STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) } - end - - KCODE_TO_ENCODING = Hash.new(Encoding::BINARY). - update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS) - - def with_encoding(enc) - before = Encoding.default_external - silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] } - yield - silence_warnings { Encoding.default_external = before } - end -end From 3fe7ca1dbea75ae83cd2eb868ba3f8518c0849a4 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:08:27 -0300 Subject: [PATCH 15/20] Replace Unicode.u_unpack with String#codepoints --- activesupport/lib/active_support/multibyte.rb | 1 - .../lib/active_support/multibyte/chars.rb | 4 ++-- .../active_support/multibyte/exceptions.rb | 8 -------- .../lib/active_support/multibyte/unicode.rb | 19 +++---------------- activesupport/test/multibyte_chars_test.rb | 11 ----------- 5 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 activesupport/lib/active_support/multibyte/exceptions.rb diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index cabe073616..fc15af17db 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -3,7 +3,6 @@ require 'active_support/core_ext/module/attribute_accessors' module ActiveSupport #:nodoc: module Multibyte - autoload :EncodingError, 'active_support/multibyte/exceptions' autoload :Chars, 'active_support/multibyte/chars' autoload :Unicode, 'active_support/multibyte/unicode' diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index 1389cb50c9..c0796320b2 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -153,7 +153,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 2 # 'é'.mb_chars.decompose.to_s.length # => 3 def decompose - chars(Unicode.decompose(:canonical, Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.decompose(:canonical, @wrapped_string.codepoints.to_a).pack('U*')) end # Performs composition on all the characters. @@ -162,7 +162,7 @@ module ActiveSupport #:nodoc: # 'é'.length # => 3 # 'é'.mb_chars.compose.to_s.length # => 2 def compose - chars(Unicode.compose(Unicode.u_unpack(@wrapped_string)).pack('U*')) + chars(Unicode.compose(@wrapped_string.codepoints.to_a).pack('U*')) end # Returns the number of grapheme clusters in the string. diff --git a/activesupport/lib/active_support/multibyte/exceptions.rb b/activesupport/lib/active_support/multibyte/exceptions.rb deleted file mode 100644 index 62066e3c71..0000000000 --- a/activesupport/lib/active_support/multibyte/exceptions.rb +++ /dev/null @@ -1,8 +0,0 @@ -# encoding: utf-8 - -module ActiveSupport #:nodoc: - module Multibyte #:nodoc: - # Raised when a problem with the encoding was found. - class EncodingError < StandardError; end - end -end \ No newline at end of file diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index c689c14631..e258e2e48e 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -61,19 +61,6 @@ module ActiveSupport TRAILERS_PAT = /(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+\Z/u LEADERS_PAT = /\A(#{codepoints_to_pattern(LEADERS_AND_TRAILERS)})+/u - # Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn't - # valid UTF-8. - # - # Example: - # Unicode.u_unpack('Café') # => [67, 97, 102, 233] - def u_unpack(string) - begin - string.unpack 'U*' - rescue ArgumentError - raise EncodingError, 'malformed UTF-8 character' - end - end - # Detect whether the codepoint is in a certain character class. Returns +true+ when it's in the specified # character class and +false+ otherwise. Valid character classes are: :cr, :lf, :l, # :v, :lv, :lvt and :t. @@ -89,7 +76,7 @@ module ActiveSupport # Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]] # Unicode.g_unpack('Café') # => [[67], [97], [102], [233]] def g_unpack(string) - codepoints = u_unpack(string) + codepoints = string.codepoints.to_a unpacked = [] pos = 0 marker = 0 @@ -283,7 +270,7 @@ module ActiveSupport def normalize(string, form=nil) form ||= @default_normalization_form # See http://www.unicode.org/reports/tr15, Table 1 - codepoints = u_unpack(string) + codepoints = string.codepoints.to_a case form when :d reorder_characters(decompose(:canonical, codepoints)) @@ -299,7 +286,7 @@ module ActiveSupport end def apply_mapping(string, mapping) #:nodoc: - u_unpack(string).map do |codepoint| + string.each_codepoint.map do |codepoint| cp = database.codepoints[codepoint] if cp and (ncp = cp.send(mapping)) and ncp > 0 ncp diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 0c6b03f15f..87830d57d3 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -72,17 +72,6 @@ class MultibyteCharsTest < Test::Unit::TestCase assert !@proxy_class.consumes?(BYTE_STRING) end - def test_unpack_utf8_strings - assert_equal 4, ActiveSupport::Multibyte::Unicode.u_unpack(UNICODE_STRING).length - assert_equal 5, ActiveSupport::Multibyte::Unicode.u_unpack(ASCII_STRING).length - end - - def test_unpack_raises_encoding_error_on_broken_strings - assert_raise(ActiveSupport::Multibyte::EncodingError) do - ActiveSupport::Multibyte::Unicode.u_unpack(BYTE_STRING) - end - end - def test_concatenation_should_return_a_proxy_class_instance assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars + 'b').class assert_equal ActiveSupport::Multibyte.proxy_class, ('a'.mb_chars << 'b').class From d2455bd8a56eeb41c346d8a0b49f4ba7475f2471 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:15:21 -0300 Subject: [PATCH 16/20] Remove unnecessary requires/encoding comment --- activesupport/lib/active_support/multibyte.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/activesupport/lib/active_support/multibyte.rb b/activesupport/lib/active_support/multibyte.rb index fc15af17db..5efe13c537 100644 --- a/activesupport/lib/active_support/multibyte.rb +++ b/activesupport/lib/active_support/multibyte.rb @@ -1,6 +1,3 @@ -# encoding: utf-8 -require 'active_support/core_ext/module/attribute_accessors' - module ActiveSupport #:nodoc: module Multibyte autoload :Chars, 'active_support/multibyte/chars' From a8a8dc4a691577c2d029be92c32f593d0fd7db75 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:15:36 -0300 Subject: [PATCH 17/20] Move include to top of class for clarity --- activesupport/lib/active_support/multibyte/chars.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index c0796320b2..b85c5e8bd1 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -35,6 +35,7 @@ module ActiveSupport #:nodoc: # # ActiveSupport::Multibyte.proxy_class = CharsForUTF32 class Chars + include Comparable attr_reader :wrapped_string alias to_s wrapped_string alias to_str wrapped_string @@ -69,8 +70,6 @@ module ActiveSupport #:nodoc: string.encoding == Encoding::UTF_8 end - include Comparable - # Works just like String#split, with the exception that the items in the resulting list are Chars # instances instead of String. This makes chaining methods easier. # From 4ac056c7f46fa6961699402d0bf296dac6a02384 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:16:35 -0300 Subject: [PATCH 18/20] Use more descriptive method names --- activesupport/lib/active_support/multibyte/chars.rb | 4 ++-- .../lib/active_support/multibyte/unicode.rb | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index b85c5e8bd1..a6256d0c47 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -90,7 +90,7 @@ module ActiveSupport #:nodoc: # Example: # 'Café'.mb_chars.reverse.to_s # => 'éfaC' def reverse - chars(Unicode.g_unpack(@wrapped_string).reverse.flatten.pack('U*')) + chars(Unicode.unpack_graphemes(@wrapped_string).reverse.flatten.pack('U*')) end # Limit the byte size of the string to a number of bytes without breaking characters. Usable @@ -170,7 +170,7 @@ module ActiveSupport #:nodoc: # 'क्षि'.mb_chars.length # => 4 # 'क्षि'.mb_chars.g_length # => 3 def g_length - Unicode.g_unpack(@wrapped_string).length + Unicode.unpack_graphemes(@wrapped_string).length end # Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string. diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index e258e2e48e..10e3a24e7f 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -73,9 +73,9 @@ module ActiveSupport # Unpack the string at grapheme boundaries. Returns a list of character lists. # # Example: - # Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]] - # Unicode.g_unpack('Café') # => [[67], [97], [102], [233]] - def g_unpack(string) + # Unicode.unpack_graphemes('क्षि') # => [[2325, 2381], [2359], [2367]] + # Unicode.unpack_graphemes('Café') # => [[67], [97], [102], [233]] + def unpack_graphemes(string) codepoints = string.codepoints.to_a unpacked = [] pos = 0 @@ -105,11 +105,11 @@ module ActiveSupport unpacked end - # Reverse operation of g_unpack. + # Reverse operation of unpack_graphemes. # # Example: - # Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि' - def g_pack(unpacked) + # Unicode.pack_graphemes(Unicode.unpack_graphemes('क्षि')) # => 'क्षि' + def pack_graphemes(unpacked) unpacked.flatten.pack('U*') end From db6eb1930d5e31d08e2cc613ad0570d0d1bb0730 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:20:15 -0300 Subject: [PATCH 19/20] Use friendlier method names for upcasing/downcasing --- .../lib/active_support/multibyte/chars.rb | 6 ++--- .../lib/active_support/multibyte/unicode.rb | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index a6256d0c47..eb7f99dd71 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -107,7 +107,7 @@ module ActiveSupport #:nodoc: # Example: # 'Laurent, où sont les tests ?'.mb_chars.upcase.to_s # => "LAURENT, OÙ SONT LES TESTS ?" def upcase - chars(Unicode.apply_mapping @wrapped_string, :uppercase_mapping) + chars Unicode.upcase(@wrapped_string) end # Convert characters in the string to lowercase. @@ -115,7 +115,7 @@ module ActiveSupport #:nodoc: # Example: # 'VĚDA A VÝZKUM'.mb_chars.downcase.to_s # => "věda a výzkum" def downcase - chars(Unicode.apply_mapping @wrapped_string, :lowercase_mapping) + chars Unicode.downcase(@wrapped_string) end # Converts the first character to uppercase and the remainder to lowercase. @@ -132,7 +132,7 @@ module ActiveSupport #:nodoc: # "ÉL QUE SE ENTERÓ".mb_chars.titleize # => "Él Que Se Enteró" # "日本語".mb_chars.titleize # => "日本語" def titleize - chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.apply_mapping $1, :uppercase_mapping }) + chars(downcase.to_s.gsub(/\b('?[\S])/u) { Unicode.upcase($1)}) end alias_method :titlecase, :titleize diff --git a/activesupport/lib/active_support/multibyte/unicode.rb b/activesupport/lib/active_support/multibyte/unicode.rb index 10e3a24e7f..94fb0a48aa 100644 --- a/activesupport/lib/active_support/multibyte/unicode.rb +++ b/activesupport/lib/active_support/multibyte/unicode.rb @@ -285,15 +285,12 @@ module ActiveSupport end.pack('U*') end - def apply_mapping(string, mapping) #:nodoc: - string.each_codepoint.map do |codepoint| - cp = database.codepoints[codepoint] - if cp and (ncp = cp.send(mapping)) and ncp > 0 - ncp - else - codepoint - end - end.pack('U*') + def downcase(string) + apply_mapping string, :lowercase_mapping + end + + def upcase(string) + apply_mapping string, :uppercase_mapping end # Holds data about a codepoint in the Unicode database @@ -361,6 +358,17 @@ module ActiveSupport private + def apply_mapping(string, mapping) #:nodoc: + string.each_codepoint.map do |codepoint| + cp = database.codepoints[codepoint] + if cp and (ncp = cp.send(mapping)) and ncp > 0 + ncp + else + codepoint + end + end.pack('U*') + end + def tidy_byte(byte) if byte < 160 [database.cp1252[byte] || byte].pack("U").unpack("C*") From 16bee7618c328ecd790db366221639661912c477 Mon Sep 17 00:00:00 2001 From: Norman Clarke Date: Thu, 5 Jan 2012 17:25:40 -0300 Subject: [PATCH 20/20] Use friendlier method name --- activesupport/lib/active_support/multibyte/chars.rb | 4 ++-- activesupport/test/multibyte_chars_test.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/activesupport/lib/active_support/multibyte/chars.rb b/activesupport/lib/active_support/multibyte/chars.rb index eb7f99dd71..99b974e4a7 100644 --- a/activesupport/lib/active_support/multibyte/chars.rb +++ b/activesupport/lib/active_support/multibyte/chars.rb @@ -168,8 +168,8 @@ module ActiveSupport #:nodoc: # # Example: # 'क्षि'.mb_chars.length # => 4 - # 'क्षि'.mb_chars.g_length # => 3 - def g_length + # 'क्षि'.mb_chars.grapheme_length # => 3 + def grapheme_length Unicode.unpack_graphemes(@wrapped_string).length end diff --git a/activesupport/test/multibyte_chars_test.rb b/activesupport/test/multibyte_chars_test.rb index 87830d57d3..e36bd22081 100644 --- a/activesupport/test/multibyte_chars_test.rb +++ b/activesupport/test/multibyte_chars_test.rb @@ -595,7 +595,7 @@ class MultibyteCharsExtrasTest < Test::Unit::TestCase else str = input end - assert_equal expected_length, chars(str).g_length + assert_equal expected_length, chars(str).grapheme_length end end