mirror of
https://github.com/rails/rails.git
synced 2022-11-09 12:12:34 -05:00
Remove multibyte utils
This is neither a public API, nor used internally, so let's remove it.
This commit is contained in:
parent
9ea34ad387
commit
51648a6fee
3 changed files with 1 additions and 140 deletions
|
@ -21,24 +21,5 @@ module ActiveSupport #:nodoc:
|
|||
def self.proxy_class
|
||||
@proxy_class ||= ActiveSupport::Multibyte::Chars
|
||||
end
|
||||
|
||||
# Regular expressions that describe valid byte sequences for a character
|
||||
VALID_CHARACTER = {
|
||||
# Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
|
||||
'UTF-8' => /\A(?:
|
||||
[\x00-\x7f] |
|
||||
[\xc2-\xdf] [\x80-\xbf] |
|
||||
\xe0 [\xa0-\xbf] [\x80-\xbf] |
|
||||
[\xe1-\xef] [\x80-\xbf] [\x80-\xbf] |
|
||||
\xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
|
||||
[\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
|
||||
\xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf])\z /xn,
|
||||
# Quick check for valid Shift-JIS characters, disregards the odd-even pairing
|
||||
'Shift_JIS' => /\A(?:
|
||||
[\x00-\x7e\xa1-\xdf] |
|
||||
[\x81-\x9f\xe0-\xef] [\x40-\x7e\x80-\x9e\x9f-\xfc])\z /xn
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
require 'active_support/multibyte/utils'
|
||||
end
|
|
@ -1,27 +0,0 @@
|
|||
# encoding: utf-8
|
||||
|
||||
module ActiveSupport #:nodoc:
|
||||
module Multibyte #:nodoc:
|
||||
# Returns a regular expression that matches valid characters in the current encoding
|
||||
def self.valid_character
|
||||
VALID_CHARACTER[Encoding.default_external.to_s]
|
||||
end
|
||||
|
||||
# Verifies the encoding of a string
|
||||
def self.verify(string)
|
||||
string.valid_encoding?
|
||||
end
|
||||
|
||||
# Verifies the encoding of the string and raises an exception when it's not valid
|
||||
def self.verify!(string)
|
||||
raise EncodingError.new("Found characters with invalid encoding") unless verify(string)
|
||||
end
|
||||
|
||||
# Removes all invalid characters from the string.
|
||||
#
|
||||
# Note: this method is a no-op in Ruby 1.9
|
||||
def self.clean(string)
|
||||
string
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,93 +0,0 @@
|
|||
# encoding: utf-8
|
||||
|
||||
require 'abstract_unit'
|
||||
require 'multibyte_test_helpers'
|
||||
|
||||
class MultibyteUtilsTest < ActiveSupport::TestCase
|
||||
include MultibyteTestHelpers
|
||||
|
||||
test "valid_character returns an expression for the current encoding" do
|
||||
with_encoding('None') do
|
||||
assert_nil ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
with_encoding('UTF8') do
|
||||
assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['UTF-8'], ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
with_encoding('SJIS') do
|
||||
assert_equal ActiveSupport::Multibyte::VALID_CHARACTER['Shift_JIS'], ActiveSupport::Multibyte.valid_character
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies ASCII strings are properly encoded" do
|
||||
with_encoding('None') do
|
||||
examples.each do |example|
|
||||
assert ActiveSupport::Multibyte.verify(example)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies UTF-8 strings are properly encoded" do
|
||||
with_encoding('UTF8') do
|
||||
assert ActiveSupport::Multibyte.verify(example('valid UTF-8'))
|
||||
assert !ActiveSupport::Multibyte.verify(example('invalid UTF-8'))
|
||||
end
|
||||
end
|
||||
|
||||
test "verify verifies Shift-JIS strings are properly encoded" do
|
||||
with_encoding('SJIS') do
|
||||
assert ActiveSupport::Multibyte.verify(example('valid Shift-JIS'))
|
||||
assert !ActiveSupport::Multibyte.verify(example('invalid Shift-JIS'))
|
||||
end
|
||||
end
|
||||
|
||||
test "verify! raises an exception when it finds an invalid character" do
|
||||
with_encoding('UTF8') do
|
||||
assert_raises(ActiveSupport::Multibyte::EncodingError) do
|
||||
ActiveSupport::Multibyte.verify!(example('invalid UTF-8'))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "verify! doesn't raise an exception when the encoding is valid" do
|
||||
with_encoding('UTF8') do
|
||||
assert_nothing_raised do
|
||||
ActiveSupport::Multibyte.verify!(example('valid UTF-8'))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "clean is a no-op" do
|
||||
with_encoding('UTF8') do
|
||||
assert_equal example('invalid Shift-JIS'), ActiveSupport::Multibyte.clean(example('invalid Shift-JIS'))
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
STRINGS = {
|
||||
'valid ASCII' => [65, 83, 67, 73, 73].pack('C*'),
|
||||
'invalid ASCII' => [128].pack('C*'),
|
||||
'valid UTF-8' => [227, 129, 147, 227, 129, 171, 227, 129, 161, 227, 130, 143].pack('C*'),
|
||||
'invalid UTF-8' => [184, 158, 8, 136, 165].pack('C*'),
|
||||
'valid Shift-JIS' => [131, 122, 129, 91, 131, 128].pack('C*'),
|
||||
'invalid Shift-JIS' => [184, 158, 8, 0, 255, 136, 165].pack('C*')
|
||||
}
|
||||
|
||||
def example(key)
|
||||
STRINGS[key].force_encoding(Encoding.default_external)
|
||||
end
|
||||
|
||||
def examples
|
||||
STRINGS.values.map { |s| s.force_encoding(Encoding.default_external) }
|
||||
end
|
||||
|
||||
KCODE_TO_ENCODING = Hash.new(Encoding::BINARY).
|
||||
update('UTF8' => Encoding::UTF_8, 'SJIS' => Encoding::Shift_JIS)
|
||||
|
||||
def with_encoding(enc)
|
||||
before = Encoding.default_external
|
||||
silence_warnings { Encoding.default_external = KCODE_TO_ENCODING[enc] }
|
||||
yield
|
||||
silence_warnings { Encoding.default_external = before }
|
||||
end
|
||||
end
|
Loading…
Reference in a new issue