Deprecate the `:tokenizer` option to `validates_length_of`

As demonstrated by #19570, this option is severely limited, and
satisfies an extremely specific use case. Realistically, there's not
much reason for this option to exist. Its functionality can be trivially
replicated with a normal Ruby method. Let's deprecate this option, in
favor of the simpler solution.

activemodel/CHANGELOG.md

@@ -1,3 +1,8 @@
+*   Deprecate the `:tokenizer` option for `validates_length_of`, in favor of
+    plain Ruby.
+
+    *Sean Griffin*
+
 *   Deprecate `ActiveModel::Errors#add_on_empty` and `ActiveModel::Errors#add_on_blank`
     with no replacement.
 

activemodel/lib/active_model/validations/length.rb

@@ -1,3 +1,5 @@
+require "active_support/core_ext/string/strip"
+
 module ActiveModel
 
   # == Active \Model Length Validator
@@ -18,6 +20,27 @@ module ActiveModel
           options[:minimum] = 1
         end
 
+        if options[:tokenizer]
+          ActiveSupport::Deprecation.warn(<<-EOS.strip_heredoc)
+            The `:tokenizer` option is deprecated, and will be removed in Rails 5.1.
+            You can achieve the same functionality be defining an instance method
+            with the value that you want to validate the length of. For example,
+
+                validates_length_of :essay, minimum: 100,
+                  tokenizer: ->(str) { str.scan(/\w+/) }
+
+            should be written as
+
+                validates_length_of :words_in_essay, minimum: 100
+
+                private
+
+                def words_in_essay
+                  essay.scan(/\w+/)
+                end
+          EOS
+        end
+
         super
       end
 
@@ -88,8 +111,13 @@ module ActiveModel
       #     validates_length_of :user_name, within: 6..20, too_long: 'pick a shorter name', too_short: 'pick a longer name'
       #     validates_length_of :zip_code, minimum: 5, too_short: 'please enter at least 5 characters'
       #     validates_length_of :smurf_leader, is: 4, message: "papa is spelled with 4 characters... don't play me."
-      #     validates_length_of :essay, minimum: 100, too_short: 'Your essay must be at least 100 words.',
-      #                         tokenizer: ->(str) { str.scan(/\w+/) }
+      #     validates_length_of :words_in_essay, minimum: 100, too_short: 'Your essay must be at least 100 words.'
+      #
+      #     private
+      #
+      #     def words_in_essay
+      #       essay.scan(/\w+/)
+      #     end
       #   end
       #
       # Configuration options:
@@ -112,12 +140,6 @@ module ActiveModel
       # * <tt>:message</tt> - The error message to use for a <tt>:minimum</tt>,
       #   <tt>:maximum</tt>, or <tt>:is</tt> violation. An alias of the appropriate
       #   <tt>too_long</tt>/<tt>too_short</tt>/<tt>wrong_length</tt> message.
-      # * <tt>:tokenizer</tt> - A method (as a symbol), proc or string to
-      #   specify how to split up the attribute string. (e.g.
-      #   <tt>tokenizer: :word_tokenizer</tt> to call the +word_tokenizer+ method
-      #   or <tt>tokenizer: ->(str) { str.scan(/\w+/) }</tt> to count words as in
-      #   above example). Defaults to <tt>->(value) { value.split(//) }</tt> which
-      #   counts individual characters.
       #
       # There is also a list of default options supported by every validator:
       # +:if+, +:unless+, +:on+ and +:strict+.

activemodel/test/cases/validations/length_validation_test.rb

@@ -319,8 +319,14 @@ class LengthValidationTest < ActiveModel::TestCase
   end
 
   def test_validates_length_of_with_block
-    Topic.validates_length_of :content, minimum: 5, too_short: "Your essay must be at least %{count} words.",
-                                        tokenizer: lambda {|str| str.scan(/\w+/) }
+    assert_deprecated do
+      Topic.validates_length_of(
+        :content,
+        minimum: 5,
+        too_short: "Your essay must be at least %{count} words.",
+        tokenizer: lambda {|str| str.scan(/\w+/) },
+      )
+    end
     t = Topic.new(content: "this content should be long enough")
     assert t.valid?
 
@@ -332,8 +338,14 @@ class LengthValidationTest < ActiveModel::TestCase
 
 
   def test_validates_length_of_with_symbol
-    Topic.validates_length_of :content, minimum: 5, too_short: "Your essay must be at least %{count} words.",
-                                        tokenizer: :my_word_tokenizer
+    assert_deprecated do
+      Topic.validates_length_of(
+        :content,
+        minimum: 5,
+        too_short: "Your essay must be at least %{count} words.",
+        tokenizer: :my_word_tokenizer,
+      )
+    end
     t = Topic.new(content: "this content should be long enough")
     assert t.valid?