Merge pull request #29504 from kirs/fixtures-arel-bulk

Use bulk INSERT to insert fixtures

Gemfile.lock

@@ -17,7 +17,7 @@ GIT
 
 GIT
   remote: https://github.com/rails/arel.git
-  revision: 5db56a513286814991c27000af2c0243cc19d1e2
+  revision: 67a51c62f4e19390cd8eb408596ca48bb0806362
   specs:
     arel (8.0.0)
 

activerecord/CHANGELOG.md

@@ -1,3 +1,7 @@
+*   Use bulk INSERT to insert fixtures for better performance.
+
+    *Kir Shatrov*
+
 *   Prevent making bind param if casted value is nil.
 
     *Ryuta Kamizono*

activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb

@@ -296,6 +296,9 @@ module ActiveRecord
 
       # Inserts the given fixture into the table. Overridden in adapters that require
       # something beyond a simple insert (eg. Oracle).
+      # Most of adapters should implement `insert_fixtures` that leverages bulk SQL insert.
+      # We keep this method to provide fallback
+      # for databases like sqlite that do not support bulk inserts.
       def insert_fixture(fixture, table_name)
         fixture = fixture.stringify_keys
 
@@ -312,12 +315,7 @@ module ActiveRecord
         table = Arel::Table.new(table_name)
 
         values = binds.map do |bind|
-          value = bind.value_for_database
-          begin
-            quote(value)
-          rescue TypeError
-            value = YAML.dump(value)
-          end
+          value = with_yaml_fallback(bind.value_for_database)
           [table[bind.name], value]
         end
 
@@ -327,6 +325,40 @@ module ActiveRecord
         execute manager.to_sql, "Fixture Insert"
       end
 
+      # Inserts a set of fixtures into the table. Overridden in adapters that require
+      # something beyond a simple insert (eg. Oracle).
+      def insert_fixtures(fixtures, table_name)
+        return if fixtures.empty?
+
+        columns = schema_cache.columns_hash(table_name)
+
+        values = fixtures.map do |fixture|
+          fixture = fixture.stringify_keys
+
+          unknown_columns = fixture.keys - columns.keys
+          if unknown_columns.any?
+            raise Fixture::FixtureError, %(table "#{table_name}" has no columns named #{unknown_columns.map(&:inspect).join(', ')}.)
+          end
+
+          columns.map do |name, column|
+            if fixture.key?(name)
+              type = lookup_cast_type_from_column(column)
+              bind = Relation::QueryAttribute.new(name, fixture[name], type)
+              with_yaml_fallback(bind.value_for_database)
+            else
+              Arel.sql("DEFAULT")
+            end
+          end
+        end
+
+        table = Arel::Table.new(table_name)
+        manager = Arel::InsertManager.new
+        manager.into(table)
+        columns.each_key { |column| manager.columns << table[column] }
+        manager.values = manager.create_values_list(values)
+        execute manager.to_sql, "Fixtures Insert"
+      end
+
       def empty_insert_statement_value
         "DEFAULT VALUES"
       end
@@ -388,6 +420,18 @@ module ActiveRecord
           end
           [relation, binds]
         end
+
+        # Fixture value is quoted by Arel, however scalar values
+        # are not quotable. In this case we want to convert
+        # the column value to YAML.
+        def with_yaml_fallback(value)
+          begin
+            quote(value)
+          rescue TypeError
+            value = YAML.dump(value)
+          end
+          value
+        end
     end
   end
 end

activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb

@@ -526,8 +526,25 @@ module ActiveRecord
         index.using == :btree || super
       end
 
+      def insert_fixtures(*)
+        without_sql_mode("NO_AUTO_VALUE_ON_ZERO") { super }
+      end
+
       private
 
+        def without_sql_mode(mode)
+          result = execute("SELECT @@SESSION.sql_mode")
+          current_mode = result.first[0]
+          return yield unless current_mode.include?(mode)
+
+          sql_mode = "REPLACE(@@sql_mode, '#{mode}', '')"
+          execute("SET @@SESSION.sql_mode = #{sql_mode}")
+          yield
+        ensure
+          sql_mode = "CONCAT(@@sql_mode, ',#{mode}')"
+          execute("SET @@SESSION.sql_mode = #{sql_mode}")
+        end
+
         def initialize_type_map(m)
           super
 

activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -349,6 +349,12 @@ module ActiveRecord
         end
       end
 
+      def insert_fixtures(rows, table_name)
+        rows.each do |row|
+          insert_fixture(row, table_name)
+        end
+      end
+
       private
 
         def table_structure(table_name)

activerecord/lib/active_record/fixtures.rb

@@ -567,9 +567,7 @@ module ActiveRecord
               end
 
               table_rows.each do |fixture_set_name, rows|
-                rows.each do |row|
-                  conn.insert_fixture(row, fixture_set_name)
-                end
+                conn.insert_fixtures(rows, fixture_set_name)
               end
 
               # Cap primary key sequences to max(pk).

activerecord/test/cases/adapters/postgresql/array_test.rb

@@ -191,6 +191,12 @@ class PostgresqlArrayTest < ActiveRecord::PostgreSQLTestCase
     assert_equal(PgArray.last.tags, tag_values)
   end
 
+  def test_insert_fixtures
+    tag_values = ["val1", "val2", "val3_with_'_multiple_quote_'_chars"]
+    @connection.insert_fixtures([{ "tags" => tag_values }], "pg_arrays")
+    assert_equal(PgArray.last.tags, tag_values)
+  end
+
   def test_attribute_for_inspect_for_array_field
     record = PgArray.new { |a| a.ratings = (1..10).to_a }
     assert_equal("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", record.attribute_for_inspect(:ratings))

activerecord/test/cases/fixtures_test.rb

@@ -54,6 +54,31 @@ class FixturesTest < ActiveRecord::TestCase
     end
   end
 
+  class InsertQuerySubscriber
+    attr_reader :events
+
+    def initialize
+      @events = []
+    end
+
+    def call(_, _, _, _, values)
+      @events << values[:sql] if values[:sql] =~ /INSERT/
+    end
+  end
+
+  if current_adapter?(:Mysql2Adapter, :PostgreSQLAdapter)
+    def test_bulk_insert
+      begin
+        subscriber = InsertQuerySubscriber.new
+        subscription = ActiveSupport::Notifications.subscribe("sql.active_record", subscriber)
+        create_fixtures("bulbs")
+        assert_equal 1, subscriber.events.size, "It takes one INSERT query to insert two fixtures"
+      ensure
+        ActiveSupport::Notifications.unsubscribe(subscription)
+      end
+    end
+  end
+
   def test_broken_yaml_exception
     badyaml = Tempfile.new ["foo", ".yml"]
     badyaml.write "a: : "
@@ -248,7 +273,12 @@ class FixturesTest < ActiveRecord::TestCase
     e = assert_raise(ActiveRecord::Fixture::FixtureError) do
       ActiveRecord::FixtureSet.create_fixtures(FIXTURES_ROOT + "/naked/yml", "parrots")
     end
-    assert_equal(%(table "parrots" has no column named "arrr".), e.message)
+
+    if current_adapter?(:SQLite3Adapter)
+      assert_equal(%(table "parrots" has no column named "arrr".), e.message)
+    else
+      assert_equal(%(table "parrots" has no columns named "arrr", "foobar".), e.message)
+    end
   end
 
   def test_yaml_file_with_symbol_columns

activerecord/test/fixtures/naked/yml/parrots.yml

@@ -1,2 +1,3 @@
 george:
   arrr: "Curious George"
+  foobar: Foobar