Merge pull request #41856 from MSNexploder/virtual-columns

Add support for generated columns in PostgreSQL (Redux)

activerecord/CHANGELOG.md

@@ -1,3 +1,18 @@
+*   Add support for generated columns in PostgreSQL adapter
+
+    Generated columns are supported since version 12.0 of PostgreSQL. This adds
+    support of those to the PostgreSQL adapter.
+
+    ```ruby
+    create_table :users do |t|
+      t.string :name
+      t.virtual :name_upcased, type: :string, as: 'upper(name)', stored: true
+    end
+    ```
+
+    *Michał Begejowicz*
+
+
 ## Rails 7.0.0.alpha2 (September 15, 2021) ##
 
 *   No changes.

activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb

@@ -469,7 +469,7 @@ module ActiveRecord
         end
 
         def build_fixture_sql(fixtures, table_name)
-          columns = schema_cache.columns_hash(table_name)
+          columns = schema_cache.columns_hash(table_name).reject { |_, column| supports_virtual_columns? && column.virtual? }
 
           values_list = fixtures.map do |fixture|
             fixture = fixture.stringify_keys

activerecord/lib/active_record/connection_adapters/postgresql/column.rb

@@ -1,20 +1,32 @@
 # frozen_string_literal: true
 
+require "active_support/core_ext/object/blank"
+
 module ActiveRecord
   module ConnectionAdapters
     module PostgreSQL
       class Column < ConnectionAdapters::Column # :nodoc:
         delegate :oid, :fmod, to: :sql_type_metadata
 
-        def initialize(*, serial: nil, **)
+        def initialize(*, serial: nil, generated: nil, **)
           super
           @serial = serial
+          @generated = generated
         end
 
         def serial?
           @serial
         end
 
+        def virtual?
+          # We assume every generated column is virtual, no matter the concrete type
+          @generated.present?
+        end
+
+        def has_default?
+          super && !virtual?
+        end
+
         def array
           sql_type_metadata.sql_type.end_with?("[]")
         end

activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb

@@ -61,6 +61,19 @@ module ActiveRecord
             if options[:collation]
               sql << " COLLATE \"#{options[:collation]}\""
             end
+
+            if as = options[:as]
+              sql << " GENERATED ALWAYS AS (#{as})"
+
+              if options[:stored]
+                sql << " STORED"
+              else
+                raise ArgumentError, <<~MSG
+                  PostgreSQL currently does not support VIRTUAL (not persisted) generated columns.
+                  Specify 'stored: true' option for '#{options[:column].name}'
+                MSG
+              end
+            end
             super
           end
 

activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb

@@ -191,6 +191,15 @@ module ActiveRecord
           @unlogged = ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.create_unlogged_tables
         end
 
+        def new_column_definition(name, type, **options) # :nodoc:
+          case type
+          when :virtual
+            type = options[:type]
+          end
+
+          super
+        end
+
         private
           def aliased_types(name, fallback)
             fallback

activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb

@@ -19,6 +19,13 @@ module ActiveRecord
           def prepare_column_options(column)
             spec = super
             spec[:array] = "true" if column.array?
+
+            if @connection.supports_virtual_columns? && column.virtual?
+              spec[:as] = extract_expression_for_virtual_column(column)
+              spec[:stored] = true
+              spec = { type: schema_type(column).inspect }.merge!(spec)
+            end
+
             spec
           end
 
@@ -43,6 +50,10 @@ module ActiveRecord
           def schema_expression(column)
             super unless column.serial?
           end
+
+          def extract_expression_for_virtual_column(column)
+            column.default_function.inspect
+          end
       end
     end
   end

activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb

@@ -654,7 +654,7 @@ module ActiveRecord
           end
 
           def new_column_from_field(table_name, field)
-            column_name, type, default, notnull, oid, fmod, collation, comment = field
+            column_name, type, default, notnull, oid, fmod, collation, comment, attgenerated = field
             type_metadata = fetch_type_metadata(column_name, type, oid.to_i, fmod.to_i)
             default_value = extract_value_from_default(default)
             default_function = extract_default_function(default_value, default)
@@ -671,7 +671,8 @@ module ActiveRecord
               default_function,
               collation: collation,
               comment: comment.presence,
-              serial: serial
+              serial: serial,
+              generated: attgenerated
             )
           end
 

activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb

@@ -181,7 +181,7 @@ module ActiveRecord
       end
 
       def supports_partitioned_indexes?
-        database_version >= 110_000
+        database_version >= 110_000 # >= 11.0
       end
 
       def supports_partial_index?
@@ -233,12 +233,16 @@ module ActiveRecord
       end
 
       def supports_insert_on_conflict?
-        database_version >= 90500
+        database_version >= 90500 # >= 9.5
       end
       alias supports_insert_on_duplicate_skip? supports_insert_on_conflict?
       alias supports_insert_on_duplicate_update? supports_insert_on_conflict?
       alias supports_insert_conflict_target? supports_insert_on_conflict?
 
+      def supports_virtual_columns?
+        database_version >= 120_000 # >= 12.0
+      end
+
       def index_algorithms
         { concurrently: "CONCURRENTLY" }
       end
@@ -388,7 +392,7 @@ module ActiveRecord
       end
 
       def supports_pgcrypto_uuid?
-        database_version >= 90400
+        database_version >= 90400 # >= 9.4
       end
 
       def supports_optimizer_hints?
@@ -489,7 +493,7 @@ module ActiveRecord
       end
 
       def check_version # :nodoc:
-        if database_version < 90300
+        if database_version < 90300 # < 9.3
           raise "Your version of PostgreSQL (#{database_version}) is too old. Active Record supports PostgreSQL >= 9.3."
         end
       end
@@ -874,7 +878,8 @@ module ActiveRecord
           query(<<~SQL, "SCHEMA")
               SELECT a.attname, format_type(a.atttypid, a.atttypmod),
                      pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod,
-                     c.collname, col_description(a.attrelid, a.attnum) AS comment
+                     c.collname, col_description(a.attrelid, a.attnum) AS comment,
+                     #{supports_virtual_columns? ? 'attgenerated' : quote('')} as attgenerated
                 FROM pg_attribute a
                 LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum
                 LEFT JOIN pg_type t ON a.atttypid = t.oid

activerecord/test/cases/adapters/postgresql/virtual_column_test.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "cases/helper"
+require "support/schema_dumping_helper"
+
+if ActiveRecord::Base.connection.supports_virtual_columns?
+  class PostgresqlVirtualColumnTest < ActiveRecord::PostgreSQLTestCase
+    include SchemaDumpingHelper
+
+    self.use_transactional_tests = false
+
+    class VirtualColumn < ActiveRecord::Base
+    end
+
+    def setup
+      @connection = ActiveRecord::Base.connection
+      @connection.create_table :virtual_columns, force: true do |t|
+        t.string  :name
+        t.virtual :upper_name,  type: :string,  as: "UPPER(name)", stored: true
+        t.virtual :name_length, type: :integer, as: "LENGTH(name)", stored: true
+        t.virtual :name_octet_length, type: :integer, as: "OCTET_LENGTH(name)", stored: true
+      end
+      VirtualColumn.create(name: "Rails")
+    end
+
+    def teardown
+      @connection.drop_table :virtual_columns, if_exists: true
+      VirtualColumn.reset_column_information
+    end
+
+    def test_virtual_column
+      column = VirtualColumn.columns_hash["upper_name"]
+      assert_predicate column, :virtual?
+      assert_equal "RAILS", VirtualColumn.take.upper_name
+    end
+
+    def test_stored_column
+      column = VirtualColumn.columns_hash["name_length"]
+      assert_predicate column, :virtual?
+      assert_equal 5, VirtualColumn.take.name_length
+    end
+
+    def test_change_table
+      @connection.change_table :virtual_columns do |t|
+        t.virtual :lower_name, type: :string, as: "LOWER(name)", stored: true
+      end
+      VirtualColumn.reset_column_information
+      column = VirtualColumn.columns_hash["lower_name"]
+      assert_predicate column, :virtual?
+      assert_equal "rails", VirtualColumn.take.lower_name
+    end
+
+    def test_non_persisted_column
+      message = <<~MSG
+        PostgreSQL currently does not support VIRTUAL (not persisted) generated columns.
+        Specify 'stored: true' option for 'invalid_definition'
+      MSG
+
+      assert_raise ArgumentError, message do
+        @connection.change_table :virtual_columns do |t|
+          t.virtual :invalid_definition, type: :string, as: "LOWER(name)"
+        end
+      end
+    end
+
+    def test_schema_dumping
+      output = dump_table_schema("virtual_columns")
+      assert_match(/t\.virtual\s+"upper_name",\s+type: :string,\s+as: "upper\(\(name\)::text\)", stored: true$/i, output)
+      assert_match(/t\.virtual\s+"name_length",\s+type: :integer,\s+as: "length\(\(name\)::text\)", stored: true$/i, output)
+      assert_match(/t\.virtual\s+"name_octet_length",\s+type: :integer,\s+as: "octet_length\(\(name\)::text\)", stored: true$/i, output)
+    end
+
+    def test_build_fixture_sql
+      ActiveRecord::FixtureSet.create_fixtures(FIXTURES_ROOT, :virtual_columns)
+    end
+  end
+end

activerecord/test/fixtures/virtual_columns.yml

@@ -0,0 +1,5 @@
+one:
+  name: hello
+
+two:
+  name: world

guides/source/active_record_postgresql.md

@@ -503,14 +503,36 @@ irb> device.id
 NOTE: `gen_random_uuid()` (from `pgcrypto`) is assumed if no `:default` option was
 passed to `create_table`.
 
+Generated Columns
+-----------------
+
+NOTE: Generated columns are supported since version 12.0 of PostgreSQL.
+
+```ruby
+# db/migrate/20131220144913_create_users.rb
+create_table :users do |t|
+  t.string :name
+  t.virtual :name_upcased, type: :string, as: 'upper(name)', stored: true
+end
+
+# app/models/user.rb
+class User < ApplicationRecord
+end
+
+# Usage
+user = User.create(name: 'John')
+User.last.name_upcased # => "JOHN"
+```
+
+
 Full Text Search
 ----------------
 
 ```ruby
 # db/migrate/20131220144913_create_documents.rb
 create_table :documents do |t|
-  t.string 'title'
-  t.string 'body'
+  t.string :title
+  t.string :body
 end
 
 add_index :documents, "to_tsvector('english', title || ' ' || body)", using: :gin, name: 'documents_idx'
@@ -531,6 +553,27 @@ Document.where("to_tsvector('english', title || ' ' || body) @@ to_tsquery(?)",
                  "cat & dog")
 ```
 
+Optionally, you can store the vector as automatically generated column (from PostgreSQL 12.0):
+
+```ruby
+# db/migrate/20131220144913_create_documents.rb
+create_table :documents do |t|
+  t.string :title
+  t.string :body
+
+  t.virtual :textsearchable_index_col,
+            type: :tsvector, as: "to_tsvector('english', title || ' ' || body)"
+end
+
+add_index :documents, :textsearchable_index_col, using: :gin, name: 'documents_idx'
+
+# Usage
+Document.create(title: "Cats and Dogs", body: "are nice!")
+
+## all documents matching 'cat & dog'
+Document.where("textsearchable_index_col @@ to_tsquery(?)", "cat & dog")
+```
+
 Database Views
 --------------