Allow serialize with a custom coder on json and array columns

We already have a test case for `serialize` with a custom coder in
`PostgresqlHstoreTest`.

https://github.com/rails/rails/blob/v5.1.3/activerecord/test/cases/adapters/postgresql/hstore_test.rb#L316-L335

activerecord/lib/active_record/attribute_methods/serialization.rb

@@ -70,7 +70,7 @@ module ActiveRecord
           end
 
           decorate_attribute_type(attr_name, :serialize) do |type|
-            if type_incompatible_with_serialize?(type)
+            if type_incompatible_with_serialize?(type, class_name_or_coder)
               raise ColumnNotSerializableError.new(attr_name, type)
             end
 
@@ -80,12 +80,9 @@ module ActiveRecord
 
         private
 
-          def type_incompatible_with_serialize?(type)
-            type.is_a?(ActiveRecord::Type::Json) ||
-            (
-              defined?(ActiveRecord::ConnectionAdapters::PostgreSQL) &&
-              type.is_a?(ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array)
-            )
+          def type_incompatible_with_serialize?(type, class_name)
+            type.is_a?(ActiveRecord::Type::Json) && class_name == ::JSON ||
+              type.respond_to?(:type_cast_array, true) && class_name == ::Array
           end
       end
     end

activerecord/test/cases/adapters/postgresql/array_test.rb

@@ -47,7 +47,7 @@ class PostgresqlArrayTest < ActiveRecord::PostgreSQLTestCase
     assert ratings_column.array?
   end
 
-  def test_not_compatible_with_serialize
+  def test_not_compatible_with_serialize_array
     new_klass = Class.new(PgArray) do
       serialize :tags, Array
     end
@@ -56,6 +56,30 @@ class PostgresqlArrayTest < ActiveRecord::PostgreSQLTestCase
     end
   end
 
+  class MyTags
+    def initialize(tags); @tags = tags end
+    def to_a; @tags end
+    def self.load(tags); new(tags) end
+    def self.dump(object); object.to_a end
+  end
+
+  def test_array_with_serialized_attributes
+    new_klass = Class.new(PgArray) do
+      serialize :tags, MyTags
+    end
+
+    new_klass.create!(tags: MyTags.new(["one", "two"]))
+    record = new_klass.first
+
+    assert_instance_of MyTags, record.tags
+    assert_equal ["one", "two"], record.tags.to_a
+
+    record.tags = MyTags.new(["three", "four"])
+    record.save!
+
+    assert_equal ["three", "four"], record.reload.tags.to_a
+  end
+
   def test_default
     @connection.add_column "pg_arrays", "score", :integer, array: true, default: [4, 4, 2]
     PgArray.reset_column_information

activerecord/test/cases/json_shared_test_cases.rb

@@ -216,7 +216,7 @@ module JSONSharedTestCases
     assert_equal true, json.payload
   end
 
-  def test_not_compatible_with_serialize_macro
+  def test_not_compatible_with_serialize_json
     new_klass = Class.new(klass) do
       serialize :payload, JSON
     end
@@ -225,6 +225,30 @@ module JSONSharedTestCases
     end
   end
 
+  class MySettings
+    def initialize(hash); @hash = hash end
+    def to_hash; @hash end
+    def self.load(hash); new(hash) end
+    def self.dump(object); object.to_hash end
+  end
+
+  def test_json_with_serialized_attributes
+    new_klass = Class.new(klass) do
+      serialize :settings, MySettings
+    end
+
+    new_klass.create!(settings: MySettings.new("one" => "two"))
+    record = new_klass.first
+
+    assert_instance_of MySettings, record.settings
+    assert_equal({ "one" => "two" }, record.settings.to_hash)
+
+    record.settings = MySettings.new("three" => "four")
+    record.save!
+
+    assert_equal({ "three" => "four" }, record.reload.settings.to_hash)
+  end
+
   private
     def klass
       JsonDataType