2018-10-22 07:00:50 +00:00
|
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
2015-10-07 10:30:10 +00:00
|
|
|
|
module Gitlab
|
|
|
|
|
module Database
|
2021-09-02 03:09:04 +00:00
|
|
|
|
DATABASE_NAMES = %w[main ci].freeze
|
|
|
|
|
|
|
|
|
|
MAIN_DATABASE_NAME = 'main'
|
2021-06-28 21:10:13 +00:00
|
|
|
|
CI_DATABASE_NAME = 'ci'
|
2021-09-09 15:09:24 +00:00
|
|
|
|
DEFAULT_POOL_HEADROOM = 10
|
2021-06-28 21:10:13 +00:00
|
|
|
|
|
2021-05-07 15:10:39 +00:00
|
|
|
|
# This constant is used when renaming tables concurrently.
|
|
|
|
|
# If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename.
|
|
|
|
|
# Example:
|
|
|
|
|
# TABLES_TO_BE_RENAMED = {
|
|
|
|
|
# 'old_name' => 'new_name'
|
|
|
|
|
# }.freeze
|
2021-07-07 15:07:24 +00:00
|
|
|
|
TABLES_TO_BE_RENAMED = {}.freeze
|
2021-05-07 15:10:39 +00:00
|
|
|
|
|
2020-07-14 21:09:03 +00:00
|
|
|
|
# Minimum PostgreSQL version requirement per documentation:
|
|
|
|
|
# https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements
|
2021-06-14 18:10:28 +00:00
|
|
|
|
MINIMUM_POSTGRES_VERSION = 12
|
2020-07-07 03:09:32 +00:00
|
|
|
|
|
2016-06-18 17:55:45 +00:00
|
|
|
|
# https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
|
|
|
|
|
MAX_INT_VALUE = 2147483647
|
2020-08-19 12:10:17 +00:00
|
|
|
|
MIN_INT_VALUE = -2147483648
|
2019-06-13 13:12:28 +00:00
|
|
|
|
|
2017-10-30 23:21:56 +00:00
|
|
|
|
# The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz:
|
|
|
|
|
# https://www.postgresql.org/docs/9.1/static/datatype-datetime.html
|
|
|
|
|
# https://dev.mysql.com/doc/refman/5.7/en/datetime.html
|
2019-06-13 13:12:28 +00:00
|
|
|
|
# FIXME: this should just be the max value of timestampz
|
2017-10-30 23:21:56 +00:00
|
|
|
|
MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze
|
2016-06-18 17:55:45 +00:00
|
|
|
|
|
2019-07-17 09:54:40 +00:00
|
|
|
|
# The maximum number of characters for text fields, to avoid DoS attacks via parsing huge text fields
|
2019-09-18 14:02:45 +00:00
|
|
|
|
# https://gitlab.com/gitlab-org/gitlab-foss/issues/61974
|
2019-07-17 09:54:40 +00:00
|
|
|
|
MAX_TEXT_SIZE_LIMIT = 1_000_000
|
|
|
|
|
|
2019-06-25 17:28:18 +00:00
|
|
|
|
# Minimum schema version from which migrations are supported
|
2019-06-20 06:35:56 +00:00
|
|
|
|
# Migrations before this version may have been removed
|
|
|
|
|
MIN_SCHEMA_VERSION = 20190506135400
|
2019-06-20 14:56:46 +00:00
|
|
|
|
MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
|
2019-06-20 06:35:56 +00:00
|
|
|
|
|
2020-06-26 15:08:45 +00:00
|
|
|
|
# Schema we store dynamically managed partitions in (e.g. for time partitioning)
|
2020-06-25 00:09:26 +00:00
|
|
|
|
DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
|
|
|
|
|
|
2020-06-26 15:08:45 +00:00
|
|
|
|
# Schema we store static partitions in (e.g. for hash partitioning)
|
|
|
|
|
STATIC_PARTITIONS_SCHEMA = :gitlab_partitions_static
|
|
|
|
|
|
2020-06-25 00:09:26 +00:00
|
|
|
|
# This is an extensive list of postgres schemas owned by GitLab
|
|
|
|
|
# It does not include the default public schema
|
2020-06-26 15:08:45 +00:00
|
|
|
|
EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
|
2020-06-25 00:09:26 +00:00
|
|
|
|
|
2021-07-23 15:09:21 +00:00
|
|
|
|
PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym
|
2021-05-11 06:10:29 +00:00
|
|
|
|
|
2021-09-22 21:11:31 +00:00
|
|
|
|
def self.database_base_models
|
|
|
|
|
@database_base_models ||= {
|
2021-10-11 15:13:01 +00:00
|
|
|
|
# Note that we use ActiveRecord::Base here and not ApplicationRecord.
|
|
|
|
|
# This is deliberate, as we also use these classes to apply load
|
|
|
|
|
# balancing to, and the load balancer must be enabled for _all_ models
|
|
|
|
|
# that inher from ActiveRecord::Base; not just our own models that
|
|
|
|
|
# inherit from ApplicationRecord.
|
|
|
|
|
main: ::ActiveRecord::Base,
|
2021-11-03 03:13:15 +00:00
|
|
|
|
ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil
|
2021-11-15 15:10:57 +00:00
|
|
|
|
}.compact.with_indifferent_access.freeze
|
2017-03-17 13:16:47 +00:00
|
|
|
|
end
|
|
|
|
|
|
2021-11-24 12:10:21 +00:00
|
|
|
|
# This returns a list of base models with connection associated for a given gitlab_schema
|
|
|
|
|
def self.schemas_to_base_models
|
|
|
|
|
@schemas_to_base_models ||= {
|
|
|
|
|
gitlab_main: [self.database_base_models.fetch(:main)],
|
|
|
|
|
gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main
|
|
|
|
|
gitlab_shared: self.database_base_models.values # all models
|
|
|
|
|
}.with_indifferent_access.freeze
|
|
|
|
|
end
|
|
|
|
|
|
2021-12-10 18:14:42 +00:00
|
|
|
|
def self.all_database_names
|
|
|
|
|
DATABASE_NAMES
|
|
|
|
|
end
|
|
|
|
|
|
2021-09-09 15:09:24 +00:00
|
|
|
|
# We configure the database connection pool size automatically based on the
|
|
|
|
|
# configured concurrency. We also add some headroom, to make sure we don't
|
|
|
|
|
# run out of connections when more threads besides the 'user-facing' ones
|
|
|
|
|
# are running.
|
|
|
|
|
#
|
|
|
|
|
# Read more about this in
|
|
|
|
|
# doc/development/database/client_side_connection_pool.md
|
|
|
|
|
def self.default_pool_size
|
|
|
|
|
headroom =
|
|
|
|
|
(ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i
|
|
|
|
|
|
|
|
|
|
Gitlab::Runtime.max_threads + headroom
|
|
|
|
|
end
|
|
|
|
|
|
2021-06-28 21:10:13 +00:00
|
|
|
|
def self.has_config?(database_name)
|
|
|
|
|
Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s)
|
|
|
|
|
end
|
|
|
|
|
|
2021-07-09 09:09:53 +00:00
|
|
|
|
def self.main_database?(name)
|
|
|
|
|
# The database is `main` if it is a first entry in `database.yml`
|
|
|
|
|
# Rails internally names them `primary` to avoid confusion
|
|
|
|
|
# with broad `primary` usage we use `main` instead
|
|
|
|
|
#
|
|
|
|
|
# TODO: The explicit `== 'main'` is needed in a transition period till
|
|
|
|
|
# the `database.yml` is not migrated into `main:` syntax
|
|
|
|
|
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65243
|
|
|
|
|
ActiveRecord::Base.configurations.primary?(name.to_s) || name.to_s == 'main'
|
|
|
|
|
end
|
|
|
|
|
|
2021-06-28 21:10:13 +00:00
|
|
|
|
def self.ci_database?(name)
|
2021-07-09 09:09:53 +00:00
|
|
|
|
name.to_s == CI_DATABASE_NAME
|
2021-06-28 21:10:13 +00:00
|
|
|
|
end
|
|
|
|
|
|
2022-02-15 18:14:39 +00:00
|
|
|
|
class PgUser < ApplicationRecord
|
|
|
|
|
self.table_name = 'pg_user'
|
|
|
|
|
self.primary_key = :usename
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
|
|
|
def self.check_for_non_superuser
|
|
|
|
|
user = PgUser.find_by('usename = CURRENT_USER')
|
|
|
|
|
am_i_superuser = user.usesuper
|
|
|
|
|
|
|
|
|
|
Gitlab::AppLogger.info(
|
|
|
|
|
"Account details: User: \"#{user.usename}\", UseSuper: (#{am_i_superuser})"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
raise 'Error: detected superuser' if am_i_superuser
|
|
|
|
|
rescue ActiveRecord::StatementInvalid
|
|
|
|
|
raise 'User CURRENT_USER not found'
|
|
|
|
|
end
|
|
|
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
|
|
2020-07-07 03:09:32 +00:00
|
|
|
|
def self.check_postgres_version_and_print_warning
|
|
|
|
|
return if Gitlab::Runtime.rails_runner?
|
|
|
|
|
|
2021-11-15 15:10:57 +00:00
|
|
|
|
database_base_models.each do |name, model|
|
|
|
|
|
database = Gitlab::Database::Reflection.new(model)
|
|
|
|
|
|
|
|
|
|
next if database.postgresql_minimum_supported_version?
|
2021-07-23 15:09:21 +00:00
|
|
|
|
|
|
|
|
|
Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result
|
|
|
|
|
|
|
|
|
|
██ ██ █████ ██████ ███ ██ ██ ███ ██ ██████
|
|
|
|
|
██ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██ ██
|
|
|
|
|
██ █ ██ ███████ ██████ ██ ██ ██ ██ ██ ██ ██ ██ ███
|
|
|
|
|
██ ███ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██
|
|
|
|
|
███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████
|
|
|
|
|
|
|
|
|
|
******************************************************************************
|
2021-11-15 15:10:57 +00:00
|
|
|
|
You are using PostgreSQL #{database.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
|
2021-07-23 15:09:21 +00:00
|
|
|
|
is required for this version of GitLab.
|
|
|
|
|
<% if Rails.env.development? || Rails.env.test? %>
|
|
|
|
|
If using gitlab-development-kit, please find the relevant steps here:
|
|
|
|
|
https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql
|
|
|
|
|
<% end %>
|
|
|
|
|
Please upgrade your environment to a supported PostgreSQL version, see
|
|
|
|
|
https://docs.gitlab.com/ee/install/requirements.html#database for details.
|
|
|
|
|
******************************************************************************
|
|
|
|
|
EOS
|
|
|
|
|
rescue ActiveRecord::ActiveRecordError, PG::Error
|
|
|
|
|
# ignore - happens when Rake tasks yet have to create a database, e.g. for testing
|
|
|
|
|
end
|
2020-07-07 03:09:32 +00:00
|
|
|
|
end
|
|
|
|
|
|
2016-06-15 10:10:41 +00:00
|
|
|
|
def self.random
|
2019-06-13 13:12:28 +00:00
|
|
|
|
"RANDOM()"
|
2016-06-15 10:10:41 +00:00
|
|
|
|
end
|
|
|
|
|
|
2017-04-10 10:23:28 +00:00
|
|
|
|
def self.true_value
|
2019-06-13 13:12:28 +00:00
|
|
|
|
"'t'"
|
2015-12-09 15:31:42 +00:00
|
|
|
|
end
|
|
|
|
|
|
2017-04-10 10:23:28 +00:00
|
|
|
|
def self.false_value
|
2019-06-13 13:12:28 +00:00
|
|
|
|
"'f'"
|
2015-12-09 15:31:42 +00:00
|
|
|
|
end
|
2016-02-02 02:29:37 +00:00
|
|
|
|
|
2017-10-30 23:21:56 +00:00
|
|
|
|
def self.sanitize_timestamp(timestamp)
|
|
|
|
|
MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
|
|
|
|
|
end
|
|
|
|
|
|
2022-03-08 12:20:17 +00:00
|
|
|
|
def self.all_uncached(&block)
|
|
|
|
|
# Calls to #uncached only disable caching for the current connection. Since the load balancer
|
|
|
|
|
# can potentially upgrade from read to read-write mode (using a different connection), we specify
|
|
|
|
|
# up-front that we'll explicitly use the primary for the duration of the operation.
|
|
|
|
|
Gitlab::Database::LoadBalancing::Session.current.use_primary do
|
|
|
|
|
base_models = database_base_models.values
|
|
|
|
|
base_models.reduce(block) { |blk, model| -> { model.uncached(&blk) } }.call
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2021-08-05 12:09:57 +00:00
|
|
|
|
def self.allow_cross_joins_across_databases(url:)
|
|
|
|
|
# this method is implemented in:
|
|
|
|
|
# spec/support/database/prevent_cross_joins.rb
|
2021-08-31 15:10:29 +00:00
|
|
|
|
yield
|
2021-08-13 09:09:58 +00:00
|
|
|
|
end
|
|
|
|
|
|
2018-09-12 01:41:14 +00:00
|
|
|
|
def self.add_post_migrate_path_to_rails(force: false)
|
|
|
|
|
return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
|
|
|
|
|
|
|
|
|
|
Rails.application.config.paths['db'].each do |db_path|
|
|
|
|
|
path = Rails.root.join(db_path, 'post_migrate').to_s
|
|
|
|
|
|
|
|
|
|
unless Rails.application.config.paths['db/migrate'].include? path
|
|
|
|
|
Rails.application.config.paths['db/migrate'] << path
|
|
|
|
|
|
|
|
|
|
# Rails memoizes migrations at certain points where it won't read the above
|
|
|
|
|
# path just yet. As such we must also update the following list of paths.
|
|
|
|
|
ActiveRecord::Migrator.migrations_paths << path
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
2019-06-25 09:15:35 +00:00
|
|
|
|
|
2021-08-06 15:10:05 +00:00
|
|
|
|
def self.db_config_names
|
2022-04-19 18:09:14 +00:00
|
|
|
|
::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) - ['geo']
|
2021-08-06 15:10:05 +00:00
|
|
|
|
end
|
|
|
|
|
|
2022-03-16 18:08:16 +00:00
|
|
|
|
# This returns all matching schemas that a given connection can use
|
|
|
|
|
# Since the `ActiveRecord::Base` might change the connection (from main to ci)
|
|
|
|
|
# This does not look at literal connection names, but rather compares
|
|
|
|
|
# models that are holders for a given db_config_name
|
|
|
|
|
def self.gitlab_schemas_for_connection(connection)
|
2022-04-08 06:08:32 +00:00
|
|
|
|
db_name = self.db_config_name(connection)
|
|
|
|
|
primary_model = self.database_base_models.fetch(db_name.to_sym)
|
|
|
|
|
|
|
|
|
|
self.schemas_to_base_models.select do |_, child_models|
|
|
|
|
|
child_models.any? do |child_model|
|
|
|
|
|
child_model == primary_model || \
|
|
|
|
|
# The model might indicate a child connection, ensure that this is enclosed in a `db_config`
|
|
|
|
|
self.database_base_models[self.db_config_share_with(child_model.connection_db_config)] == primary_model
|
|
|
|
|
end
|
|
|
|
|
end.keys.map!(&:to_sym)
|
2022-03-16 18:08:16 +00:00
|
|
|
|
end
|
|
|
|
|
|
2021-09-15 12:11:13 +00:00
|
|
|
|
def self.db_config_for_connection(connection)
|
|
|
|
|
return unless connection
|
|
|
|
|
|
2022-03-16 18:08:16 +00:00
|
|
|
|
if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy)
|
|
|
|
|
return connection.load_balancer.configuration.primary_db_config
|
|
|
|
|
end
|
2021-09-15 12:11:13 +00:00
|
|
|
|
|
|
|
|
|
# During application init we might receive `NullPool`
|
|
|
|
|
return unless connection.respond_to?(:pool) &&
|
|
|
|
|
connection.pool.respond_to?(:db_config)
|
|
|
|
|
|
|
|
|
|
connection.pool.db_config
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# At the moment, the connection can only be retrieved by
|
|
|
|
|
# Gitlab::Database::LoadBalancer#read or #read_write or from the
|
|
|
|
|
# ActiveRecord directly. Therefore, if the load balancer doesn't
|
|
|
|
|
# recognize the connection, this method returns the primary role
|
|
|
|
|
# directly. In future, we may need to check for other sources.
|
|
|
|
|
# Expected returned names:
|
|
|
|
|
# main, main_replica, ci, ci_replica, unknown
|
|
|
|
|
def self.db_config_name(connection)
|
|
|
|
|
db_config = db_config_for_connection(connection)
|
|
|
|
|
db_config&.name || 'unknown'
|
2021-06-22 00:07:53 +00:00
|
|
|
|
end
|
|
|
|
|
|
2022-03-24 12:07:26 +00:00
|
|
|
|
# Currently the database configuration can only be shared with `main:`
|
|
|
|
|
# If the `database_tasks: false` is being used
|
|
|
|
|
# This is to be refined: https://gitlab.com/gitlab-org/gitlab/-/issues/356580
|
|
|
|
|
def self.db_config_share_with(db_config)
|
|
|
|
|
if db_config.database_tasks?
|
|
|
|
|
nil # no sharing
|
|
|
|
|
else
|
|
|
|
|
'main' # share with `main:`
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2021-08-02 09:10:09 +00:00
|
|
|
|
def self.read_only?
|
|
|
|
|
false
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def self.read_write?
|
|
|
|
|
!read_only?
|
|
|
|
|
end
|
|
|
|
|
|
2021-08-13 03:10:09 +00:00
|
|
|
|
# Monkeypatch rails with upgraded database observability
|
|
|
|
|
def self.install_transaction_metrics_patches!
|
|
|
|
|
ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def self.install_transaction_context_patches!
|
|
|
|
|
ActiveRecord::ConnectionAdapters::TransactionManager
|
|
|
|
|
.prepend(TransactionManagerContext)
|
|
|
|
|
ActiveRecord::ConnectionAdapters::RealTransaction
|
|
|
|
|
.prepend(RealTransactionContext)
|
|
|
|
|
end
|
|
|
|
|
|
2019-06-21 09:18:19 +00:00
|
|
|
|
# MonkeyPatch for ActiveRecord::Base for adding observability
|
|
|
|
|
module ActiveRecordBaseTransactionMetrics
|
2021-03-10 15:09:11 +00:00
|
|
|
|
extend ActiveSupport::Concern
|
|
|
|
|
|
|
|
|
|
class_methods do
|
2021-08-24 15:10:36 +00:00
|
|
|
|
# A patch over ActiveRecord::Base.transaction that provides
|
|
|
|
|
# observability into transactional methods.
|
2021-03-10 15:09:11 +00:00
|
|
|
|
def transaction(**options, &block)
|
2021-11-03 15:13:48 +00:00
|
|
|
|
transaction_type = get_transaction_type(connection.transaction_open?, options[:requires_new])
|
|
|
|
|
|
|
|
|
|
::Gitlab::Database::Metrics.subtransactions_increment(self.name) if transaction_type == :sub_transaction
|
|
|
|
|
|
|
|
|
|
payload = { connection: connection, transaction_type: transaction_type }
|
2021-08-24 15:10:36 +00:00
|
|
|
|
|
2021-11-03 15:13:48 +00:00
|
|
|
|
ActiveSupport::Notifications.instrument('transaction.active_record', payload) do
|
2021-03-10 15:09:11 +00:00
|
|
|
|
super(**options, &block)
|
|
|
|
|
end
|
|
|
|
|
end
|
2021-11-03 15:13:48 +00:00
|
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
|
|
def get_transaction_type(transaction_open, requires_new_flag)
|
|
|
|
|
if transaction_open
|
|
|
|
|
return :sub_transaction if requires_new_flag
|
|
|
|
|
|
|
|
|
|
return :fake_transaction
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
:real_transaction
|
|
|
|
|
end
|
2019-06-21 09:18:19 +00:00
|
|
|
|
end
|
|
|
|
|
end
|
2021-08-13 03:10:09 +00:00
|
|
|
|
|
|
|
|
|
# rubocop:disable Gitlab/ModuleWithInstanceVariables
|
|
|
|
|
module TransactionManagerContext
|
|
|
|
|
def transaction_context
|
|
|
|
|
@stack.first.try(:gitlab_transaction_context)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
module RealTransactionContext
|
|
|
|
|
def gitlab_transaction_context
|
|
|
|
|
@gitlab_transaction_context ||= ::Gitlab::Database::Transaction::Context.new
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def commit
|
|
|
|
|
gitlab_transaction_context.commit
|
|
|
|
|
|
|
|
|
|
super
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def rollback
|
|
|
|
|
gitlab_transaction_context.rollback
|
|
|
|
|
|
|
|
|
|
super
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
# rubocop:enable Gitlab/ModuleWithInstanceVariables
|
2015-10-07 10:30:10 +00:00
|
|
|
|
end
|
|
|
|
|
end
|
2021-08-02 09:10:09 +00:00
|
|
|
|
|
|
|
|
|
Gitlab::Database.prepend_mod_with('Gitlab::Database')
|