kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-10-21 00:12:27 +00:00
parent 7263a0140a
commit 551734207f
45 changed files with 898 additions and 1444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.rubocop_manual_todo.yml
View File

@ -2445,11 +2445,6 @@ Database/MultipleDatabases:
- 'lib/gitlab/database/migrations/observers/query_log.rb'
- 'lib/gitlab/database/multi_threaded_migration.rb'
- 'lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb'
- 'lib/gitlab/database/partitioning/monthly_strategy.rb'
- 'lib/gitlab/database/partitioning/partition_manager.rb'
- 'lib/gitlab/database/partitioning/partition_creator.rb'
- 'lib/gitlab/database/partitioning/replace_table.rb'
- 'lib/gitlab/database/partitioning/time_partition.rb'
- 'lib/gitlab/database/postgres_hll/batch_distinct_counter.rb'
- 'lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb'
- 'lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb'

2
app/assets/javascripts/emoji/index.js
View File

@ -10,7 +10,7 @@ let validEmojiNames = null;
export const FALLBACK_EMOJI_KEY = 'grey_question';
// Keep the version in sync with `lib/gitlab/emoji.rb`
export const EMOJI_VERSION = '2';
export const EMOJI_VERSION = '1';
const isLocalStorageAvailable = AccessorUtilities.canUseLocalStorage();

41
app/assets/javascripts/group_settings/components/shared_runners_form.vue
View File

@ -3,13 +3,7 @@ import { GlToggle, GlLoadingIcon, GlTooltip, GlAlert } from '@gitlab/ui';
import { debounce } from 'lodash';
import axios from '~/lib/utils/axios_utils';
import { __ } from '~/locale';
import {
DEBOUNCE_TOGGLE_DELAY,
ERROR_MESSAGE,
ENABLED,
DISABLED,
ALLOW_OVERRIDE,
} from '../constants';
import { DEBOUNCE_TOGGLE_DELAY, ERROR_MESSAGE } from '../constants';
export default {
components: {
@ -18,21 +12,14 @@ export default {
GlTooltip,
GlAlert,
},
props: {
updatePath: {
type: String,
required: true,
},
sharedRunnersAvailability: {
type: String,
required: true,
},
parentSharedRunnersAvailability: {
type: String,
required: false,
default: '',
},
},
inject: [
'updatePath',
'sharedRunnersAvailability',
'parentSharedRunnersAvailability',
'runnerEnabled',
'runnerDisabled',
'runnerAllowOverride',
],
data() {
return {
isLoading: false,
@ -43,21 +30,21 @@ export default {
},
computed: {
toggleDisabled() {
return this.parentSharedRunnersAvailability === DISABLED || this.isLoading;
return this.parentSharedRunnersAvailability === this.runnerDisabled || this.isLoading;
},
enabledOrDisabledSetting() {
return this.enabled ? ENABLED : DISABLED;
return this.enabled ? this.runnerEnabled : this.runnerDisabled;
},
disabledWithOverrideSetting() {
return this.allowOverride ? ALLOW_OVERRIDE : DISABLED;
return this.allowOverride ? this.runnerAllowOverride : this.runnerDisabled;
},
},
created() {
if (this.sharedRunnersAvailability !== ENABLED) {
if (this.sharedRunnersAvailability !== this.runnerEnabled) {
this.enabled = false;
}
if (this.sharedRunnersAvailability === ALLOW_OVERRIDE) {
if (this.sharedRunnersAvailability === this.runnerAllowOverride) {
this.allowOverride = true;
}
},

5
app/assets/javascripts/group_settings/constants.js
View File

@ -4,8 +4,3 @@ import { __ } from '~/locale';
export const DEBOUNCE_TOGGLE_DELAY = 1000;
export const ERROR_MESSAGE = __('Refresh the page and try again.');
// runner setting options
export const ENABLED = 'enabled';
export const DISABLED = 'disabled_and_unoverridable';
export const ALLOW_OVERRIDE = 'disabled_with_override';

18
app/assets/javascripts/group_settings/mount_shared_runners.js
View File

@ -4,11 +4,27 @@ import UpdateSharedRunnersForm from './components/shared_runners_form.vue';
export default (containerId = 'update-shared-runners-form') => {
const containerEl = document.getElementById(containerId);
const {
updatePath,
sharedRunnersAvailability,
parentSharedRunnersAvailability,
runnerEnabled,
runnerDisabled,
runnerAllowOverride,
} = containerEl.dataset;
return new Vue({
el: containerEl,
render(createElement) {
return createElement(UpdateSharedRunnersForm, {
props: containerEl.dataset,
provide: {
updatePath,
sharedRunnersAvailability,
parentSharedRunnersAvailability,
runnerEnabled,
runnerDisabled,
runnerAllowOverride,
},
});
},
});

5
app/helpers/ci/runners_helper.rb
View File

@ -61,7 +61,10 @@ module Ci
{
update_path: api_v4_groups_path(id: group.id),
shared_runners_availability: group.shared_runners_setting,
parent_shared_runners_availability: group.parent&.shared_runners_setting
parent_shared_runners_availability: group.parent&.shared_runners_setting,
runner_enabled: Namespace::SR_ENABLED,
runner_disabled: Namespace::SR_DISABLED_AND_UNOVERRIDABLE,
runner_allow_override: Namespace::SR_DISABLED_WITH_OVERRIDE
}
end

1
app/models/application_record.rb
View File

@ -1,7 +1,6 @@
# frozen_string_literal: true
class ApplicationRecord < ActiveRecord::Base
self.gitlab_schema = :gitlab_main
self.abstract_class = true
alias_method :reset, :reload

1
app/models/ci/application_record.rb
View File

@ -2,7 +2,6 @@
module Ci
class ApplicationRecord < ::ApplicationRecord
self.gitlab_schema = :gitlab_ci
self.abstract_class = true
def self.table_name_prefix

2
app/models/concerns/loose_foreign_key.rb
View File

@ -69,7 +69,7 @@ module LooseForeignKey
end
on_delete_options = %i[async_delete async_nullify]
gitlab_schema_options = [ApplicationRecord.gitlab_schema, Ci::ApplicationRecord.gitlab_schema]
gitlab_schema_options = %i(gitlab_main gitlab_ci)
unless on_delete_options.include?(symbolized_options[:on_delete]&.to_sym)
raise "Invalid on_delete option given: #{symbolized_options[:on_delete]}. Valid options: #{on_delete_options.join(', ')}"

2
app/workers/database/drop_detached_partitions_worker.rb
View File

@ -12,7 +12,7 @@ module Database
def perform
Gitlab::Database::Partitioning.drop_detached_partitions
ensure
Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics
Gitlab::Database::Partitioning.report_metrics
end
end
end

2
app/workers/database/partition_management_worker.rb
View File

@ -14,7 +14,7 @@ module Database
def perform
Gitlab::Database::Partitioning.sync_partitions
ensure
Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics
Gitlab::Database::Partitioning.report_metrics
end
end
end

10
config/initializers/00_active_record_gitlab_schema.rb
View File

@ -1,10 +0,0 @@
# frozen_string_literal: true
# This parameter describes a virtual context to indicate
# table affinity to other tables.
#
# Table affinity limits cross-joins, cross-modifications,
# foreign keys and validates relationship between tables
#
# By default it is undefined
ActiveRecord::Base.class_attribute :gitlab_schema, default: nil

4
config/initializers/0_acts_as_taggable.rb
View File

@ -13,7 +13,3 @@ raise "Counter cache is not disabled" if
ActsAsTaggableOn::Tagging.include IgnorableColumns
ActsAsTaggableOn::Tagging.ignore_column :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
ActsAsTaggableOn::Tagging.ignore_column :taggable_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
# The tags and taggings are supposed to be part of `gitlab_ci`
ActsAsTaggableOn::Tag.gitlab_schema = :gitlab_ci
ActsAsTaggableOn::Tagging.gitlab_schema = :gitlab_ci

1
db/migrate/20210629031900_associate_existing_dast_builds_with_variables.rb
View File

@ -20,7 +20,6 @@ class AssociateExistingDastBuildsWithVariables < ActiveRecord::Migration[6.1]
class Build < ApplicationRecord
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
self.gitlab_schema = :gitlab_ci
default_scope { where(name: :dast, stage: :dast) } # rubocop:disable Cop/DefaultScope
end

2
doc/ci/pipelines/settings.md
View File

@ -78,7 +78,7 @@ To avoid this scenario:
1. Select the **Skip outdated deployment jobs** checkbox.
1. Select **Save changes**.
Older deployment job are skipped when a new deployment starts.
Older deployment jobs are skipped when a new deployment starts.
For more information, see [Deployment safety](../environments/deployment_safety.md).

BIN
doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v14_2.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 45 KiB

BIN
doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v14_4.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 123 KiB

14
doc/user/application_security/security_dashboard/index.md
View File

@ -51,7 +51,7 @@ The security dashboard and vulnerability report displays information about vulne
At the pipeline level, the Security section displays the vulnerabilities present in the branch of
the project the pipeline ran against.
![Pipeline Security Dashboard](img/pipeline_security_dashboard_v14_2.png)
![Pipeline Security Dashboard](img/pipeline_security_dashboard_v14_4.png)
Visit the page for any pipeline that ran any of the [supported reports](#supported-reports). To view
the pipeline's security findings, select the **Security** tab when viewing the pipeline.
@ -64,11 +64,15 @@ the analyzer outputs an
### Scan details
> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3728) in GitLab 13.10.
> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3728) in GitLab 13.10.
> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/333660) in GitLab 14.2.
The **Scan details** section lists the scans run in the pipeline and the total number of
vulnerabilities per scan. For the DAST scan, select **Download scanned resources** to download a
CSV file containing details of the resources scanned.
The **Scan details** section lists the scans run in the pipeline and the total number of vulnerabilities
per scan.
You can download the JSON artifacts from each security scan. Select **Download results** then
select the JSON artifact. Additionally for the DAST scan, from the **Download results** dropdown select
**Download scanned resources** to download a CSV file containing details of the resources scanned.
## Project Security Dashboard

39
lib/gitlab/database/each_database.rb Normal file
View File

@ -0,0 +1,39 @@
# frozen_string_literal: true
module Gitlab
module Database
module EachDatabase
class << self
def each_database_connection
Gitlab::Database.databases.each_pair do |connection_name, connection_wrapper|
connection = connection_wrapper.scope.connection
with_shared_connection(connection, connection_name) do
yield connection, connection_name
end
end
end
def each_model_connection(models)
models.each do |model|
connection_name = model.connection.pool.db_config.name
with_shared_connection(model.connection, connection_name) do
yield model, connection_name
end
end
end
private
def with_shared_connection(connection, connection_name)
Gitlab::Database::SharedModel.using_connection(connection) do
Gitlab::AppLogger.debug(message: 'Switched database connection', connection_name: connection_name)
yield
end
end
end
end
end
end

24
lib/gitlab/database/partitioning.rb
View File

@ -12,11 +12,31 @@ module Gitlab
end
def self.sync_partitions(models_to_sync = registered_models)
MultiDatabasePartitionManager.new(models_to_sync).sync_partitions
Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions')
Gitlab::Database::EachDatabase.each_model_connection(models_to_sync) do |model|
PartitionManager.new(model).sync_partitions
end
Gitlab::AppLogger.info(message: 'Finished sync of dynamic postgres partitions')
end
def self.report_metrics(models_to_monitor = registered_models)
partition_monitoring = PartitionMonitoring.new
Gitlab::Database::EachDatabase.each_model_connection(models_to_monitor) do |model|
partition_monitoring.report_metrics_for_model(model)
end
end
def self.drop_detached_partitions
MultiDatabasePartitionDropper.new.drop_detached_partitions
Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions')
Gitlab::Database::EachDatabase.each_database_connection do
DetachedPartitionDropper.new.perform
end
Gitlab::AppLogger.info(message: 'Finished dropping detached postgres partitions')
end
end
end

4
lib/gitlab/database/partitioning/monthly_strategy.rb
View File

@ -96,10 +96,6 @@ module Gitlab
def oldest_active_date
(Date.today - retain_for).beginning_of_month
end
def connection
ActiveRecord::Base.connection
end
end
end
end

35
lib/gitlab/database/partitioning/multi_database_partition_dropper.rb
View File

@ -1,35 +0,0 @@
# frozen_string_literal: true
module Gitlab
module Database
module Partitioning
class MultiDatabasePartitionDropper
def drop_detached_partitions
Gitlab::AppLogger.info(message: "Dropping detached postgres partitions")
each_database_connection do |name, connection|
Gitlab::Database::SharedModel.using_connection(connection) do
Gitlab::AppLogger.debug(message: "Switched database connection", connection_name: name)
DetachedPartitionDropper.new.perform
end
end
Gitlab::AppLogger.info(message: "Finished dropping detached postgres partitions")
end
private
def each_database_connection
databases.each_pair do |name, connection_wrapper|
yield name, connection_wrapper.scope.connection
end
end
def databases
Gitlab::Database.databases
end
end
end
end
end

37
lib/gitlab/database/partitioning/multi_database_partition_manager.rb
View File

@ -1,37 +0,0 @@
# frozen_string_literal: true
module Gitlab
module Database
module Partitioning
class MultiDatabasePartitionManager
def initialize(models)
@models = models
end
def sync_partitions
Gitlab::AppLogger.info(message: "Syncing dynamic postgres partitions")
models.each do |model|
Gitlab::Database::SharedModel.using_connection(model.connection) do
Gitlab::AppLogger.debug(message: "Switched database connection",
connection_name: connection_name,
table_name: model.table_name)
PartitionManager.new(model).sync_partitions
end
end
Gitlab::AppLogger.info(message: "Finished sync of dynamic postgres partitions")
end
private
attr_reader :models
def connection_name
Gitlab::Database::SharedModel.connection.pool.db_config.name
end
end
end
end
end

18
lib/gitlab/database/partitioning/partition_monitoring.rb
View File

@ -4,20 +4,12 @@ module Gitlab
module Database
module Partitioning
class PartitionMonitoring
attr_reader :models
def report_metrics_for_model(model)
strategy = model.partitioning_strategy
def initialize(models = Gitlab::Database::Partitioning.registered_models)
@models = models
end
def report_metrics
models.each do |model|
strategy = model.partitioning_strategy
gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
end
gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
end
private

7
lib/gitlab/database/partitioning/replace_table.rb
View File

@ -9,7 +9,8 @@ module Gitlab
attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column,
:sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key
def initialize(original_table, replacement_table, replaced_table, primary_key_column)
def initialize(connection, original_table, replacement_table, replaced_table, primary_key_column)
@connection = connection
@original_table = original_table
@replacement_table = replacement_table
@replaced_table = replaced_table
@ -29,10 +30,8 @@ module Gitlab
private
attr_reader :connection
delegate :execute, :quote_table_name, :quote_column_name, to: :connection
def connection
@connection ||= ActiveRecord::Base.connection
end
def default_sequence(table, column)
"#{table}_#{column}_seq"

2
lib/gitlab/database/partitioning/time_partition.rb
View File

@ -87,7 +87,7 @@ module Gitlab
end
def conn
@conn ||= ActiveRecord::Base.connection
@conn ||= Gitlab::Database::SharedModel.connection
end
end
end

4
lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
View File

@ -428,8 +428,8 @@ module Gitlab
end
def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name)
replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s,
replacement_table_name, replaced_table_name, primary_key_name)
replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(connection,
original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name)
transaction do
drop_sync_trigger(original_table_name)

2
lib/gitlab/emoji.rb
View File

@ -6,7 +6,7 @@ module Gitlab
# When updating emoji assets increase the version below
# and update the version number in `app/assets/javascripts/emoji/index.js`
EMOJI_VERSION = 2
EMOJI_VERSION = 1
# Return a Pathname to emoji's current versioned folder
#

2
lib/gitlab/zentao/client.rb
View File

@ -16,7 +16,7 @@ module Gitlab
def ping
response = fetch_product(zentao_product_xid) rescue {}
active = response.fetch('deleted') == '0' rescue false
active = response['deleted'] == '0'
if active
{ success: true }
else

47
spec/frontend/group_settings/components/shared_runners_form_spec.js
View File

@ -3,13 +3,16 @@ import { shallowMount } from '@vue/test-utils';
import MockAxiosAdapter from 'axios-mock-adapter';
import waitForPromises from 'helpers/wait_for_promises';
import SharedRunnersForm from '~/group_settings/components/shared_runners_form.vue';
import { ENABLED, DISABLED, ALLOW_OVERRIDE } from '~/group_settings/constants';
import axios from '~/lib/utils/axios_utils';
const TEST_UPDATE_PATH = '/test/update';
const DISABLED_PAYLOAD = { shared_runners_setting: DISABLED };
const ENABLED_PAYLOAD = { shared_runners_setting: ENABLED };
const OVERRIDE_PAYLOAD = { shared_runners_setting: ALLOW_OVERRIDE };
const provide = {
updatePath: '/test/update',
sharedRunnersAvailability: 'enabled',
parentSharedRunnersAvailability: null,
runnerDisabled: 'disabled',
runnerEnabled: 'enabled',
runnerAllowOverride: 'allow_override',
};
jest.mock('~/flash');
@ -17,13 +20,11 @@ describe('group_settings/components/shared_runners_form', () => {
let wrapper;
let mock;
const createComponent = (props = {}) => {
const createComponent = (provides = {}) => {
wrapper = shallowMount(SharedRunnersForm, {
propsData: {
updatePath: TEST_UPDATE_PATH,
sharedRunnersAvailability: ENABLED,
parentSharedRunnersAvailability: null,
...props,
provide: {
...provide,
...provides,
},
});
};
@ -33,13 +34,13 @@ describe('group_settings/components/shared_runners_form', () => {
const findEnabledToggle = () => wrapper.find('[data-testid="enable-runners-toggle"]');
const findOverrideToggle = () => wrapper.find('[data-testid="override-runners-toggle"]');
const changeToggle = (toggle) => toggle.vm.$emit('change', !toggle.props('value'));
const getRequestPayload = () => JSON.parse(mock.history.put[0].data);
const getSharedRunnersSetting = () => JSON.parse(mock.history.put[0].data).shared_runners_setting;
const isLoadingIconVisible = () => findLoadingIcon().exists();
beforeEach(() => {
mock = new MockAxiosAdapter(axios);
mock.onPut(TEST_UPDATE_PATH).reply(200);
mock.onPut(provide.updatePath).reply(200);
});
afterEach(() => {
@ -95,7 +96,7 @@ describe('group_settings/components/shared_runners_form', () => {
await waitForPromises();
expect(getRequestPayload()).toEqual(ENABLED_PAYLOAD);
expect(getSharedRunnersSetting()).toEqual(provide.runnerEnabled);
expect(findOverrideToggle().exists()).toBe(false);
});
@ -104,14 +105,14 @@ describe('group_settings/components/shared_runners_form', () => {
await waitForPromises();
expect(getRequestPayload()).toEqual(DISABLED_PAYLOAD);
expect(getSharedRunnersSetting()).toEqual(provide.runnerDisabled);
expect(findOverrideToggle().exists()).toBe(true);
});
});
describe('override toggle', () => {
beforeEach(() => {
createComponent({ sharedRunnersAvailability: ALLOW_OVERRIDE });
createComponent({ sharedRunnersAvailability: provide.runnerAllowOverride });
});
it('enabling the override toggle sends correct payload', async () => {
@ -119,7 +120,7 @@ describe('group_settings/components/shared_runners_form', () => {
await waitForPromises();
expect(getRequestPayload()).toEqual(OVERRIDE_PAYLOAD);
expect(getSharedRunnersSetting()).toEqual(provide.runnerAllowOverride);
});
it('disabling the override toggle sends correct payload', async () => {
@ -127,21 +128,21 @@ describe('group_settings/components/shared_runners_form', () => {
await waitForPromises();
expect(getRequestPayload()).toEqual(DISABLED_PAYLOAD);
expect(getSharedRunnersSetting()).toEqual(provide.runnerDisabled);
});
});
describe('toggle disabled state', () => {
it(`toggles are not disabled with setting ${DISABLED}`, () => {
createComponent({ sharedRunnersAvailability: DISABLED });
it(`toggles are not disabled with setting ${provide.runnerDisabled}`, () => {
createComponent({ sharedRunnersAvailability: provide.runnerDisabled });
expect(findEnabledToggle().props('disabled')).toBe(false);
expect(findOverrideToggle().props('disabled')).toBe(false);
});
it('toggles are disabled', () => {
createComponent({
sharedRunnersAvailability: DISABLED,
parentSharedRunnersAvailability: DISABLED,
sharedRunnersAvailability: provide.runnerDisabled,
parentSharedRunnersAvailability: provide.runnerDisabled,
});
expect(findEnabledToggle().props('disabled')).toBe(true);
expect(findOverrideToggle().props('disabled')).toBe(true);
@ -154,7 +155,7 @@ describe('group_settings/components/shared_runners_form', () => {
${{ error: 'Undefined error' }} | ${'Undefined error Refresh the page and try again.'}
`(`with error $errorObj`, ({ errorObj, message }) => {
beforeEach(async () => {
mock.onPut(TEST_UPDATE_PATH).reply(500, errorObj);
mock.onPut(provide.updatePath).reply(500, errorObj);
createComponent();
changeToggle(findEnabledToggle());

32
spec/helpers/ci/runners_helper_spec.rb
View File

@ -68,23 +68,35 @@ RSpec.describe Ci::RunnersHelper do
end
describe '#group_shared_runners_settings_data' do
let(:group) { create(:group, parent: parent, shared_runners_enabled: false) }
let(:parent) { create(:group) }
let_it_be(:parent) { create(:group) }
let_it_be(:group) { create(:group, parent: parent, shared_runners_enabled: false) }
let(:runner_constants) do
{
runner_enabled: Namespace::SR_ENABLED,
runner_disabled: Namespace::SR_DISABLED_AND_UNOVERRIDABLE,
runner_allow_override: Namespace::SR_DISABLED_WITH_OVERRIDE
}
end
it 'returns group data for top level group' do
data = group_shared_runners_settings_data(parent)
result = {
update_path: "/api/v4/groups/#{parent.id}",
shared_runners_availability: Namespace::SR_ENABLED,
parent_shared_runners_availability: nil
}.merge(runner_constants)
expect(data[:update_path]).to eq("/api/v4/groups/#{parent.id}")
expect(data[:shared_runners_availability]).to eq('enabled')
expect(data[:parent_shared_runners_availability]).to eq(nil)
expect(group_shared_runners_settings_data(parent)).to eq result
end
it 'returns group data for child group' do
data = group_shared_runners_settings_data(group)
result = {
update_path: "/api/v4/groups/#{group.id}",
shared_runners_availability: Namespace::SR_DISABLED_AND_UNOVERRIDABLE,
parent_shared_runners_availability: Namespace::SR_ENABLED
}.merge(runner_constants)
expect(data[:update_path]).to eq("/api/v4/groups/#{group.id}")
expect(data[:shared_runners_availability]).to eq(Namespace::SR_DISABLED_AND_UNOVERRIDABLE)
expect(data[:parent_shared_runners_availability]).to eq('enabled')
expect(group_shared_runners_settings_data(group)).to eq result
end
end

48
spec/lib/gitlab/database/each_database_spec.rb Normal file
View File

@ -0,0 +1,48 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::EachDatabase do
describe '.each_database' do
let(:expected_connections) do
Gitlab::Database.databases.map { |name, wrapper| [wrapper.scope.connection, name] }
end
it 'yields each connection after connecting SharedModel' do
expected_connections.each do |connection, _|
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection).and_yield
end
yielded_connections = []
described_class.each_database_connection do |connection, name|
yielded_connections << [connection, name]
end
expect(yielded_connections).to match_array(expected_connections)
end
end
describe '.each_model_connection' do
let(:model1) { double(connection: double, table_name: 'table1') }
let(:model2) { double(connection: double, table_name: 'table2') }
before do
allow(model1.connection).to receive_message_chain('pool.db_config.name').and_return('name1')
allow(model2.connection).to receive_message_chain('pool.db_config.name').and_return('name2')
end
it 'yields each model after connecting SharedModel' do
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model1.connection).and_yield
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model2.connection).and_yield
yielded_models = []
described_class.each_model_connection([model1, model2]) do |model, name|
yielded_models << [model, name]
end
expect(yielded_models).to match_array([[model1, 'name1'], [model2, 'name2']])
end
end
end

38
spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb
View File

@ -1,38 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::Partitioning::MultiDatabasePartitionDropper, '#drop_detached_partitions' do
subject(:drop_detached_partitions) { multi_db_dropper.drop_detached_partitions }
let(:multi_db_dropper) { described_class.new }
let(:connection_wrapper1) { double(scope: scope1) }
let(:connection_wrapper2) { double(scope: scope2) }
let(:scope1) { double(connection: connection1) }
let(:scope2) { double(connection: connection2) }
let(:connection1) { double('connection') }
let(:connection2) { double('connection') }
let(:dropper_class) { Gitlab::Database::Partitioning::DetachedPartitionDropper }
let(:dropper1) { double('partition dropper') }
let(:dropper2) { double('partition dropper') }
before do
allow(multi_db_dropper).to receive(:databases).and_return({ db1: connection_wrapper1, db2: connection_wrapper2 })
end
it 'drops detached partitions for each database' do
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection1).and_yield.ordered
expect(dropper_class).to receive(:new).and_return(dropper1).ordered
expect(dropper1).to receive(:perform)
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection2).and_yield.ordered
expect(dropper_class).to receive(:new).and_return(dropper2).ordered
expect(dropper2).to receive(:perform)
drop_detached_partitions
end
end

36
spec/lib/gitlab/database/partitioning/multi_database_partition_manager_spec.rb
View File

@ -1,36 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::Partitioning::MultiDatabasePartitionManager, '#sync_partitions' do
subject(:sync_partitions) { manager.sync_partitions }
let(:manager) { described_class.new(models) }
let(:models) { [model1, model2] }
let(:model1) { double('model1', connection: connection1, table_name: 'table1') }
let(:model2) { double('model2', connection: connection1, table_name: 'table2') }
let(:connection1) { double('connection1') }
let(:connection2) { double('connection2') }
let(:target_manager_class) { Gitlab::Database::Partitioning::PartitionManager }
let(:target_manager1) { double('partition manager') }
let(:target_manager2) { double('partition manager') }
before do
allow(manager).to receive(:connection_name).and_return('name')
end
it 'syncs model partitions, setting up the appropriate connection for each', :aggregate_failures do
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model1.connection).and_yield.ordered
expect(target_manager_class).to receive(:new).with(model1).and_return(target_manager1).ordered
expect(target_manager1).to receive(:sync_partitions)
expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model2.connection).and_yield.ordered
expect(target_manager_class).to receive(:new).with(model2).and_return(target_manager2).ordered
expect(target_manager2).to receive(:sync_partitions)
sync_partitions
end
end

3
spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb
View File

@ -4,9 +4,8 @@ require 'spec_helper'
RSpec.describe Gitlab::Database::Partitioning::PartitionMonitoring do
describe '#report_metrics' do
subject { described_class.new(models).report_metrics }
subject { described_class.new.report_metrics_for_model(model) }
let(:models) { [model] }
let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table) }
let(:partitioning_strategy) { double(missing_partitions: missing_partitions, current_partitions: current_partitions, extra_partitions: extra_partitions) }
let(:table) { "some_table" }

4
spec/lib/gitlab/database/partitioning/replace_table_spec.rb
View File

@ -5,7 +5,9 @@ require 'spec_helper'
RSpec.describe Gitlab::Database::Partitioning::ReplaceTable, '#perform' do
include Database::TableSchemaHelpers
subject(:replace_table) { described_class.new(original_table, replacement_table, archived_table, 'id').perform }
subject(:replace_table) do
described_class.new(connection, original_table, replacement_table, archived_table, 'id').perform
end
let(:original_table) { '_test_original_table' }
let(:replacement_table) { '_test_replacement_table' }

113
spec/lib/gitlab/database/partitioning_spec.rb
View File

@ -3,46 +3,121 @@
require 'spec_helper'
RSpec.describe Gitlab::Database::Partitioning do
include Database::PartitioningHelpers
include Database::TableSchemaHelpers
let(:connection) { ApplicationRecord.connection }
describe '.sync_partitions' do
let(:partition_manager_class) { described_class::MultiDatabasePartitionManager }
let(:partition_manager) { double('partition manager') }
let(:table_names) { %w[partitioning_test1 partitioning_test2] }
let(:models) do
table_names.map do |table_name|
Class.new(ApplicationRecord) do
include PartitionedTable
self.table_name = table_name
partitioned_by :created_at, strategy: :monthly
end
end
end
before do
table_names.each do |table_name|
connection.execute(<<~SQL)
CREATE TABLE #{table_name} (
id serial not null,
created_at timestamptz not null,
PRIMARY KEY (id, created_at))
PARTITION BY RANGE (created_at);
SQL
end
end
it 'manages partitions for each given model' do
expect { described_class.sync_partitions(models)}
.to change { find_partitions(table_names.first).size }.from(0)
.and change { find_partitions(table_names.last).size }.from(0)
end
context 'when no partitioned models are given' do
it 'calls the partition manager with the registered models' do
expect(partition_manager_class).to receive(:new)
.with(described_class.registered_models)
.and_return(partition_manager)
let(:partition_manager_class) { described_class::PartitionManager }
let(:partition_manager) { double('partition manager') }
let(:model) { double('model') }
it 'manages partitions for each registered model' do
expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
.with(described_class.registered_models)
.and_yield(model)
expect(partition_manager_class).to receive(:new).with(model).and_return(partition_manager)
expect(partition_manager).to receive(:sync_partitions)
described_class.sync_partitions
end
end
end
describe '.report_metrics' do
let(:model1) { double('model') }
let(:model2) { double('model') }
let(:partition_monitoring_class) { described_class::PartitionMonitoring }
context 'when no partitioned models are given' do
it 'reports metrics for each registered model' do
expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
end
expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
.with(described_class.registered_models)
.and_yield(model1)
.and_yield(model2)
described_class.report_metrics
end
end
context 'when partitioned models are given' do
it 'calls the partition manager with the given models' do
models = ['my special model']
it 'reports metrics for each given model' do
expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
end
expect(partition_manager_class).to receive(:new)
.with(models)
.and_return(partition_manager)
expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
.with([model1, model2])
.and_yield(model1)
.and_yield(model2)
expect(partition_manager).to receive(:sync_partitions)
described_class.sync_partitions(models)
described_class.report_metrics([model1, model2])
end
end
end
describe '.drop_detached_partitions' do
let(:partition_dropper_class) { described_class::MultiDatabasePartitionDropper }
let(:table_names) { %w[detached_test_partition1 detached_test_partition2] }
it 'delegates to the partition dropper' do
expect_next_instance_of(partition_dropper_class) do |partition_dropper|
expect(partition_dropper).to receive(:drop_detached_partitions)
before do
table_names.each do |table_name|
connection.create_table("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}")
Postgresql::DetachedPartition.create!(table_name: table_name, drop_after: 1.year.ago)
end
end
described_class.drop_detached_partitions
it 'drops detached partitions for each database' do
expect(Gitlab::Database::EachDatabase).to receive(:each_database_connection).and_yield
expect { described_class.drop_detached_partitions }
.to change { Postgresql::DetachedPartition.count }.from(2).to(0)
.and change { table_exists?(table_names.first) }.from(true).to(false)
.and change { table_exists?(table_names.last) }.from(true).to(false)
end
def table_exists?(table_name)
table_oid(table_name).present?
end
end

1133
spec/support/database/cross-database-modification-allowlist.yml
View File

File diff suppressed because it is too large Load Diff

1
spec/support/database/cross-join-allowlist.yml
View File

@ -30,6 +30,7 @@
- "./spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb"
- "./spec/migrations/20210907211557_finalize_ci_builds_bigint_conversion_spec.rb"
- "./spec/migrations/associate_existing_dast_builds_with_variables_spec.rb"
- "./spec/migrations/disable_job_token_scope_when_unused_spec.rb"
- "./spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb"
- "./spec/migrations/schedule_pages_metadata_migration_spec.rb"
- "./spec/models/ci/pipeline_spec.rb"

11
spec/support/database/gitlab_schema.rb
View File

@ -9,17 +9,12 @@ module Database
end
def self.table_schema(name)
tables_to_schema[name] || :undefined
# When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database
tables_to_schema[name] || :"undefined_#{name}"
end
def self.tables_to_schema
@tables_to_schema ||= all_classes_with_schema.to_h do |klass|
[klass.table_name, klass.gitlab_schema]
end
end
def self.all_classes_with_schema
ActiveRecord::Base.descendants.reject(&:abstract_class?).select(&:gitlab_schema?) # rubocop:disable Database/MultipleDatabases
@tables_to_schema ||= YAML.load_file(Rails.root.join('spec/support/database/gitlab_schemas.yml'))
end
end
end

536
spec/support/database/gitlab_schemas.yml Normal file
View File

@ -0,0 +1,536 @@
abuse_reports: :gitlab_main
agent_group_authorizations: :gitlab_main
agent_project_authorizations: :gitlab_main
alert_management_alert_assignees: :gitlab_main
alert_management_alerts: :gitlab_main
alert_management_alert_user_mentions: :gitlab_main
alert_management_http_integrations: :gitlab_main
allowed_email_domains: :gitlab_main
analytics_cycle_analytics_group_stages: :gitlab_main
analytics_cycle_analytics_group_value_streams: :gitlab_main
analytics_cycle_analytics_issue_stage_events: :gitlab_main
analytics_cycle_analytics_merge_request_stage_events: :gitlab_main
analytics_cycle_analytics_project_stages: :gitlab_main
analytics_cycle_analytics_project_value_streams: :gitlab_main
analytics_cycle_analytics_stage_event_hashes: :gitlab_main
analytics_devops_adoption_segments: :gitlab_main
analytics_devops_adoption_snapshots: :gitlab_main
analytics_language_trend_repository_languages: :gitlab_main
analytics_usage_trends_measurements: :gitlab_main
appearances: :gitlab_main
application_settings: :gitlab_main
application_setting_terms: :gitlab_main
approval_merge_request_rules_approved_approvers: :gitlab_main
approval_merge_request_rules: :gitlab_main
approval_merge_request_rules_groups: :gitlab_main
approval_merge_request_rule_sources: :gitlab_main
approval_merge_request_rules_users: :gitlab_main
approval_project_rules: :gitlab_main
approval_project_rules_groups: :gitlab_main
approval_project_rules_protected_branches: :gitlab_main
approval_project_rules_users: :gitlab_main
approvals: :gitlab_main
approver_groups: :gitlab_main
approvers: :gitlab_main
atlassian_identities: :gitlab_main
audit_events_external_audit_event_destinations: :gitlab_main
audit_events: :gitlab_main
authentication_events: :gitlab_main
award_emoji: :gitlab_main
aws_roles: :gitlab_main
background_migration_jobs: :gitlab_main
badges: :gitlab_main
banned_users: :gitlab_main
batched_background_migration_jobs: :gitlab_main
batched_background_migrations: :gitlab_main
board_assignees: :gitlab_main
board_group_recent_visits: :gitlab_main
board_labels: :gitlab_main
board_project_recent_visits: :gitlab_main
boards_epic_board_labels: :gitlab_main
boards_epic_board_positions: :gitlab_main
boards_epic_board_recent_visits: :gitlab_main
boards_epic_boards: :gitlab_main
boards_epic_lists: :gitlab_main
boards_epic_list_user_preferences: :gitlab_main
boards_epic_user_preferences: :gitlab_main
boards: :gitlab_main
board_user_preferences: :gitlab_main
broadcast_messages: :gitlab_main
bulk_import_configurations: :gitlab_main
bulk_import_entities: :gitlab_main
bulk_import_exports: :gitlab_main
bulk_import_export_uploads: :gitlab_main
bulk_import_failures: :gitlab_main
bulk_imports: :gitlab_main
bulk_import_trackers: :gitlab_main
chat_names: :gitlab_main
chat_teams: :gitlab_main
ci_build_needs: :gitlab_ci
ci_build_pending_states: :gitlab_ci
ci_build_report_results: :gitlab_ci
ci_builds: :gitlab_ci
ci_builds_metadata: :gitlab_ci
ci_builds_runner_session: :gitlab_ci
ci_build_trace_chunks: :gitlab_ci
ci_build_trace_metadata: :gitlab_ci
ci_daily_build_group_report_results: :gitlab_ci
ci_deleted_objects: :gitlab_ci
ci_freeze_periods: :gitlab_ci
ci_group_variables: :gitlab_ci
ci_instance_variables: :gitlab_ci
ci_job_artifacts: :gitlab_ci
ci_job_token_project_scope_links: :gitlab_ci
ci_job_variables: :gitlab_ci
ci_minutes_additional_packs: :gitlab_ci
ci_namespace_monthly_usages: :gitlab_ci
ci_pending_builds: :gitlab_ci
ci_pipeline_artifacts: :gitlab_ci
ci_pipeline_chat_data: :gitlab_ci
ci_pipeline_messages: :gitlab_ci
ci_pipeline_schedules: :gitlab_ci
ci_pipeline_schedule_variables: :gitlab_ci
ci_pipelines_config: :gitlab_ci
ci_pipelines: :gitlab_ci
ci_pipeline_variables: :gitlab_ci
ci_platform_metrics: :gitlab_ci
ci_project_monthly_usages: :gitlab_ci
ci_refs: :gitlab_ci
ci_resource_groups: :gitlab_ci
ci_resources: :gitlab_ci
ci_runner_namespaces: :gitlab_ci
ci_runner_projects: :gitlab_ci
ci_runners: :gitlab_ci
ci_running_builds: :gitlab_ci
ci_sources_pipelines: :gitlab_ci
ci_sources_projects: :gitlab_ci
ci_stages: :gitlab_ci
ci_subscriptions_projects: :gitlab_ci
ci_trigger_requests: :gitlab_ci
ci_triggers: :gitlab_ci
ci_unit_test_failures: :gitlab_ci
ci_unit_tests: :gitlab_ci
ci_variables: :gitlab_ci
cluster_agents: :gitlab_main
cluster_agent_tokens: :gitlab_main
cluster_groups: :gitlab_main
cluster_platforms_kubernetes: :gitlab_main
cluster_projects: :gitlab_main
cluster_providers_aws: :gitlab_main
cluster_providers_gcp: :gitlab_main
clusters_applications_cert_managers: :gitlab_main
clusters_applications_cilium: :gitlab_main
clusters_applications_crossplane: :gitlab_main
clusters_applications_elastic_stacks: :gitlab_main
clusters_applications_helm: :gitlab_main
clusters_applications_ingress: :gitlab_main
clusters_applications_jupyter: :gitlab_main
clusters_applications_knative: :gitlab_main
clusters_applications_prometheus: :gitlab_main
clusters_applications_runners: :gitlab_main
clusters: :gitlab_main
clusters_integration_elasticstack: :gitlab_main
clusters_integration_prometheus: :gitlab_main
clusters_kubernetes_namespaces: :gitlab_main
commit_user_mentions: :gitlab_main
compliance_management_frameworks: :gitlab_main
container_expiration_policies: :gitlab_main
container_repositories: :gitlab_main
conversational_development_index_metrics: :gitlab_main
coverage_fuzzing_corpuses: :gitlab_main
csv_issue_imports: :gitlab_main
custom_emoji: :gitlab_main
customer_relations_contacts: :gitlab_main
customer_relations_organizations: :gitlab_main
dast_profile_schedules: :gitlab_main
dast_profiles: :gitlab_main
dast_profiles_pipelines: :gitlab_main
dast_scanner_profiles_builds: :gitlab_main
dast_scanner_profiles: :gitlab_main
dast_site_profiles_builds: :gitlab_main
dast_site_profile_secret_variables: :gitlab_main
dast_site_profiles: :gitlab_main
dast_site_profiles_pipelines: :gitlab_main
dast_sites: :gitlab_main
dast_site_tokens: :gitlab_main
dast_site_validations: :gitlab_main
dep_ci_build_trace_section_names: :gitlab_main
dep_ci_build_trace_sections: :gitlab_main
dependency_proxy_blobs: :gitlab_main
dependency_proxy_group_settings: :gitlab_main
dependency_proxy_image_ttl_group_policies: :gitlab_main
dependency_proxy_manifests: :gitlab_main
deploy_keys_projects: :gitlab_main
deployment_clusters: :gitlab_main
deployment_merge_requests: :gitlab_main
deployments: :gitlab_main
deploy_tokens: :gitlab_main
description_versions: :gitlab_main
design_management_designs: :gitlab_main
design_management_designs_versions: :gitlab_main
design_management_versions: :gitlab_main
design_user_mentions: :gitlab_main
detached_partitions: :gitlab_main
diff_note_positions: :gitlab_main
dora_daily_metrics: :gitlab_main
draft_notes: :gitlab_main
elastic_index_settings: :gitlab_main
elastic_reindexing_slices: :gitlab_main
elastic_reindexing_subtasks: :gitlab_main
elastic_reindexing_tasks: :gitlab_main
elasticsearch_indexed_namespaces: :gitlab_main
elasticsearch_indexed_projects: :gitlab_main
emails: :gitlab_main
environments: :gitlab_main
epic_issues: :gitlab_main
epic_metrics: :gitlab_main
epics: :gitlab_main
epic_user_mentions: :gitlab_main
error_tracking_client_keys: :gitlab_main
error_tracking_error_events: :gitlab_main
error_tracking_errors: :gitlab_main
events: :gitlab_main
evidences: :gitlab_main
experiments: :gitlab_main
experiment_subjects: :gitlab_main
experiment_users: :gitlab_main
external_approval_rules: :gitlab_main
external_approval_rules_protected_branches: :gitlab_main
external_pull_requests: :gitlab_main
external_status_checks: :gitlab_main
external_status_checks_protected_branches: :gitlab_main
feature_gates: :gitlab_main
features: :gitlab_main
fork_network_members: :gitlab_main
fork_networks: :gitlab_main
geo_cache_invalidation_events: :gitlab_main
geo_container_repository_updated_events: :gitlab_main
geo_event_log: :gitlab_main
geo_events: :gitlab_main
geo_hashed_storage_attachments_events: :gitlab_main
geo_hashed_storage_migrated_events: :gitlab_main
geo_job_artifact_deleted_events: :gitlab_main
geo_lfs_object_deleted_events: :gitlab_main
geo_node_namespace_links: :gitlab_main
geo_nodes: :gitlab_main
geo_node_statuses: :gitlab_main
geo_repositories_changed_events: :gitlab_main
geo_repository_created_events: :gitlab_main
geo_repository_deleted_events: :gitlab_main
geo_repository_renamed_events: :gitlab_main
geo_repository_updated_events: :gitlab_main
geo_reset_checksum_events: :gitlab_main
geo_upload_deleted_events: :gitlab_main
gitlab_subscription_histories: :gitlab_main
gitlab_subscriptions: :gitlab_main
gpg_keys: :gitlab_main
gpg_key_subkeys: :gitlab_main
gpg_signatures: :gitlab_main
grafana_integrations: :gitlab_main
group_custom_attributes: :gitlab_main
group_deletion_schedules: :gitlab_main
group_deploy_keys: :gitlab_main
group_deploy_keys_groups: :gitlab_main
group_deploy_tokens: :gitlab_main
group_group_links: :gitlab_main
group_import_states: :gitlab_main
group_merge_request_approval_settings: :gitlab_main
group_repository_storage_moves: :gitlab_main
group_wiki_repositories: :gitlab_main
historical_data: :gitlab_main
identities: :gitlab_main
import_export_uploads: :gitlab_main
import_failures: :gitlab_main
incident_management_escalation_policies: :gitlab_main
incident_management_escalation_rules: :gitlab_main
incident_management_issuable_escalation_statuses: :gitlab_main
incident_management_oncall_participants: :gitlab_main
incident_management_oncall_rotations: :gitlab_main
incident_management_oncall_schedules: :gitlab_main
incident_management_oncall_shifts: :gitlab_main
incident_management_pending_alert_escalations: :gitlab_main
incident_management_pending_issue_escalations: :gitlab_main
index_statuses: :gitlab_main
in_product_marketing_emails: :gitlab_main
insights: :gitlab_main
integrations: :gitlab_main
internal_ids: :gitlab_main
ip_restrictions: :gitlab_main
issuable_metric_images: :gitlab_main
issuable_severities: :gitlab_main
issuable_slas: :gitlab_main
issue_assignees: :gitlab_main
issue_customer_relations_contacts: :gitlab_main
issue_email_participants: :gitlab_main
issue_links: :gitlab_main
issue_metrics: :gitlab_main
issues: :gitlab_main
issues_prometheus_alert_events: :gitlab_main
issues_self_managed_prometheus_alert_events: :gitlab_main
issue_tracker_data: :gitlab_main
issue_user_mentions: :gitlab_main
iterations_cadences: :gitlab_main
jira_connect_installations: :gitlab_main
jira_connect_subscriptions: :gitlab_main
jira_imports: :gitlab_main
jira_tracker_data: :gitlab_main
keys: :gitlab_main
label_links: :gitlab_main
label_priorities: :gitlab_main
labels: :gitlab_main
ldap_group_links: :gitlab_main
lfs_file_locks: :gitlab_main
lfs_objects: :gitlab_main
lfs_objects_projects: :gitlab_main
licenses: :gitlab_main
lists: :gitlab_main
list_user_preferences: :gitlab_main
loose_foreign_keys_deleted_records: :gitlab_main
members: :gitlab_main
merge_request_assignees: :gitlab_main
merge_request_blocks: :gitlab_main
merge_request_cleanup_schedules: :gitlab_main
merge_request_context_commit_diff_files: :gitlab_main
merge_request_context_commits: :gitlab_main
merge_request_diff_commits: :gitlab_main
merge_request_diff_commit_users: :gitlab_main
merge_request_diff_details: :gitlab_main
merge_request_diff_files: :gitlab_main
merge_request_diffs: :gitlab_main
merge_request_metrics: :gitlab_main
merge_request_reviewers: :gitlab_main
merge_requests_closing_issues: :gitlab_main
merge_requests: :gitlab_main
merge_request_user_mentions: :gitlab_main
merge_trains: :gitlab_main
metrics_dashboard_annotations: :gitlab_main
metrics_users_starred_dashboards: :gitlab_main
milestone_releases: :gitlab_main
milestones: :gitlab_main
namespace_admin_notes: :gitlab_main
namespace_aggregation_schedules: :gitlab_main
namespace_limits: :gitlab_main
namespace_package_settings: :gitlab_main
namespace_root_storage_statistics: :gitlab_main
namespace_settings: :gitlab_main
namespaces: :gitlab_main
namespace_statistics: :gitlab_main
note_diff_files: :gitlab_main
notes: :gitlab_main
notification_settings: :gitlab_main
oauth_access_grants: :gitlab_main
oauth_access_tokens: :gitlab_main
oauth_applications: :gitlab_main
oauth_openid_requests: :gitlab_main
onboarding_progresses: :gitlab_main
open_project_tracker_data: :gitlab_main
operations_feature_flags_clients: :gitlab_main
operations_feature_flag_scopes: :gitlab_main
operations_feature_flags: :gitlab_main
operations_feature_flags_issues: :gitlab_main
operations_scopes: :gitlab_main
operations_strategies: :gitlab_main
operations_strategies_user_lists: :gitlab_main
operations_user_lists: :gitlab_main
packages_build_infos: :gitlab_main
packages_composer_cache_files: :gitlab_main
packages_composer_metadata: :gitlab_main
packages_conan_file_metadata: :gitlab_main
packages_conan_metadata: :gitlab_main
packages_debian_file_metadata: :gitlab_main
packages_debian_group_architectures: :gitlab_main
packages_debian_group_component_files: :gitlab_main
packages_debian_group_components: :gitlab_main
packages_debian_group_distribution_keys: :gitlab_main
packages_debian_group_distributions: :gitlab_main
packages_debian_project_architectures: :gitlab_main
packages_debian_project_component_files: :gitlab_main
packages_debian_project_components: :gitlab_main
packages_debian_project_distribution_keys: :gitlab_main
packages_debian_project_distributions: :gitlab_main
packages_debian_publications: :gitlab_main
packages_dependencies: :gitlab_main
packages_dependency_links: :gitlab_main
packages_events: :gitlab_main
packages_helm_file_metadata: :gitlab_main
packages_maven_metadata: :gitlab_main
packages_nuget_dependency_link_metadata: :gitlab_main
packages_nuget_metadata: :gitlab_main
packages_package_file_build_infos: :gitlab_main
packages_package_files: :gitlab_main
packages_packages: :gitlab_main
packages_pypi_metadata: :gitlab_main
packages_rubygems_metadata: :gitlab_main
packages_tags: :gitlab_main
pages_deployments: :gitlab_main
pages_domain_acme_orders: :gitlab_main
pages_domains: :gitlab_main
partitioned_foreign_keys: :gitlab_main
path_locks: :gitlab_main
personal_access_tokens: :gitlab_main
plan_limits: :gitlab_main
plans: :gitlab_main
pool_repositories: :gitlab_main
postgres_async_indexes: :gitlab_main
postgres_reindex_actions: :gitlab_main
product_analytics_events_experimental: :gitlab_main
programming_languages: :gitlab_main
project_access_tokens: :gitlab_main
project_alerting_settings: :gitlab_main
project_aliases: :gitlab_main
project_authorizations: :gitlab_main
project_auto_devops: :gitlab_main
project_ci_cd_settings: :gitlab_main
project_ci_feature_usages: :gitlab_main
project_compliance_framework_settings: :gitlab_main
project_custom_attributes: :gitlab_main
project_daily_statistics: :gitlab_main
project_deploy_tokens: :gitlab_main
project_error_tracking_settings: :gitlab_main
project_export_jobs: :gitlab_main
project_features: :gitlab_main
project_feature_usages: :gitlab_main
project_group_links: :gitlab_main
project_import_data: :gitlab_main
project_incident_management_settings: :gitlab_main
project_metrics_settings: :gitlab_main
project_mirror_data: :gitlab_main
project_pages_metadata: :gitlab_main
project_repositories: :gitlab_main
project_repository_states: :gitlab_main
project_repository_storage_moves: :gitlab_main
project_security_settings: :gitlab_main
project_settings: :gitlab_main
projects: :gitlab_main
project_statistics: :gitlab_main
project_topics: :gitlab_main
project_tracing_settings: :gitlab_main
prometheus_alert_events: :gitlab_main
prometheus_alerts: :gitlab_main
prometheus_metrics: :gitlab_main
protected_branches: :gitlab_main
protected_branch_merge_access_levels: :gitlab_main
protected_branch_push_access_levels: :gitlab_main
protected_branch_unprotect_access_levels: :gitlab_main
protected_environment_deploy_access_levels: :gitlab_main
protected_environments: :gitlab_main
protected_tag_create_access_levels: :gitlab_main
protected_tags: :gitlab_main
push_event_payloads: :gitlab_main
push_rules: :gitlab_main
raw_usage_data: :gitlab_main
redirect_routes: :gitlab_main
release_links: :gitlab_main
releases: :gitlab_main
remote_mirrors: :gitlab_main
repository_languages: :gitlab_main
required_code_owners_sections: :gitlab_main
requirements: :gitlab_main
requirements_management_test_reports: :gitlab_main
resource_iteration_events: :gitlab_main
resource_label_events: :gitlab_main
resource_milestone_events: :gitlab_main
resource_state_events: :gitlab_main
resource_weight_events: :gitlab_main
reviews: :gitlab_main
routes: :gitlab_main
saml_group_links: :gitlab_main
saml_providers: :gitlab_main
scim_identities: :gitlab_main
scim_oauth_access_tokens: :gitlab_main
security_findings: :gitlab_main
security_orchestration_policy_configurations: :gitlab_main
security_orchestration_policy_rule_schedules: :gitlab_main
security_scans: :gitlab_main
self_managed_prometheus_alert_events: :gitlab_main
sent_notifications: :gitlab_main
sentry_issues: :gitlab_main
serverless_domain_cluster: :gitlab_main
service_desk_settings: :gitlab_main
shards: :gitlab_main
slack_integrations: :gitlab_main
smartcard_identities: :gitlab_main
snippet_repositories: :gitlab_main
snippet_repository_storage_moves: :gitlab_main
snippets: :gitlab_main
snippet_statistics: :gitlab_main
snippet_user_mentions: :gitlab_main
software_license_policies: :gitlab_main
software_licenses: :gitlab_main
spam_logs: :gitlab_main
sprints: :gitlab_main
status_check_responses: :gitlab_main
status_page_published_incidents: :gitlab_main
status_page_settings: :gitlab_main
subscriptions: :gitlab_main
suggestions: :gitlab_main
system_note_metadata: :gitlab_main
taggings: :gitlab_ci
tags: :gitlab_ci
term_agreements: :gitlab_main
terraform_states: :gitlab_main
terraform_state_versions: :gitlab_main
timelogs: :gitlab_main
todos: :gitlab_main
token_with_ivs: :gitlab_main
topics: :gitlab_main
trending_projects: :gitlab_main
u2f_registrations: :gitlab_main
upcoming_reconciliations: :gitlab_main
uploads: :gitlab_main
user_agent_details: :gitlab_main
user_callouts: :gitlab_main
user_canonical_emails: :gitlab_main
user_credit_card_validations: :gitlab_main
user_custom_attributes: :gitlab_main
user_details: :gitlab_main
user_follow_users: :gitlab_main
user_group_callouts: :gitlab_main
user_highest_roles: :gitlab_main
user_interacted_projects: :gitlab_main
user_permission_export_uploads: :gitlab_main
user_preferences: :gitlab_main
users: :gitlab_main
users_ops_dashboard_projects: :gitlab_main
users_security_dashboard_projects: :gitlab_main
users_star_projects: :gitlab_main
users_statistics: :gitlab_main
user_statuses: :gitlab_main
user_synced_attributes_metadata: :gitlab_main
verification_codes: :gitlab_main
vulnerabilities: :gitlab_main
vulnerability_exports: :gitlab_main
vulnerability_external_issue_links: :gitlab_main
vulnerability_feedback: :gitlab_main
vulnerability_finding_evidence_assets: :gitlab_main
vulnerability_finding_evidence_headers: :gitlab_main
vulnerability_finding_evidence_requests: :gitlab_main
vulnerability_finding_evidence_responses: :gitlab_main
vulnerability_finding_evidences: :gitlab_main
vulnerability_finding_evidence_sources: :gitlab_main
vulnerability_finding_evidence_supporting_messages: :gitlab_main
vulnerability_finding_links: :gitlab_main
vulnerability_finding_signatures: :gitlab_main
vulnerability_findings_remediations: :gitlab_main
vulnerability_flags: :gitlab_main
vulnerability_historical_statistics: :gitlab_main
vulnerability_identifiers: :gitlab_main
vulnerability_issue_links: :gitlab_main
vulnerability_occurrence_identifiers: :gitlab_main
vulnerability_occurrence_pipelines: :gitlab_main
vulnerability_occurrences: :gitlab_main
vulnerability_remediations: :gitlab_main
vulnerability_scanners: :gitlab_main
vulnerability_statistics: :gitlab_main
vulnerability_user_mentions: :gitlab_main
webauthn_registrations: :gitlab_main
web_hook_logs: :gitlab_main
web_hooks: :gitlab_main
wiki_page_meta: :gitlab_main
wiki_page_slugs: :gitlab_main
work_item_types: :gitlab_main
x509_certificates: :gitlab_main
x509_commit_signatures: :gitlab_main
x509_issuers: :gitlab_main
zentao_tracker_data: :gitlab_main
zoom_meetings: :gitlab_main

5
spec/support/database/prevent_cross_database_modification.rb
View File

@ -91,6 +91,11 @@ module Database
return if tables.empty?
# All migrations will write to schema_migrations in the same transaction.
# It's safe to ignore this since schema_migrations exists in all
# databases
return if tables == ['schema_migrations']
cross_database_context[:modified_tables_by_db][database].merge(tables)
all_tables = cross_database_context[:modified_tables_by_db].values.map(&:to_a).flatten

29
spec/support_specs/database/gitlab_schema_spec.rb Normal file
View File

@ -0,0 +1,29 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Database::GitlabSchema do
it 'matches all the tables in the database', :aggregate_failures do
# These tables do not need a gitlab_schema
excluded_tables = %w(ar_internal_metadata schema_migrations)
all_tables_in_database = ApplicationRecord.connection.tables
all_tables_with_gitlab_schema = described_class.tables_to_schema.keys
missing = []
all_tables_in_database.each do |table_in_database|
next if table_in_database.in?(excluded_tables)
missing << table_in_database unless all_tables_with_gitlab_schema.include?(table_in_database)
end
extras = []
all_tables_with_gitlab_schema.each do |table_with_gitlab_schema|
extras << table_with_gitlab_schema unless all_tables_in_database.include?(table_with_gitlab_schema)
end
expect(missing).to be_empty, "Missing table(s) #{missing} not found in #{described_class}.tables_to_schema. Any new tables must be added to spec/support/database/gitlab_schemas.yml ."
expect(extras).to be_empty, "Extra table(s) #{extras} found in #{described_class}.tables_to_schema. Any removed or renamed tables must be removed from spec/support/database/gitlab_schemas.yml ."
end
end

8
spec/workers/database/drop_detached_partitions_worker_spec.rb
View File

@ -6,21 +6,19 @@ RSpec.describe Database::DropDetachedPartitionsWorker do
describe '#perform' do
subject { described_class.new.perform }
let(:monitoring) { instance_double('PartitionMonitoring', report_metrics: nil) }
before do
allow(Gitlab::Database::Partitioning).to receive(:drop_detached_partitions)
allow(Gitlab::Database::Partitioning::PartitionMonitoring).to receive(:new).and_return(monitoring)
allow(Gitlab::Database::Partitioning).to receive(:report_metrics)
end
it 'delegates to Partitioning.drop_detached_partitions' do
it 'drops detached partitions' do
expect(Gitlab::Database::Partitioning).to receive(:drop_detached_partitions)
subject
end
it 'reports partition metrics' do
expect(monitoring).to receive(:report_metrics)
expect(Gitlab::Database::Partitioning).to receive(:report_metrics)
subject
end

9
spec/workers/database/partition_management_worker_spec.rb
View File

@ -6,20 +6,19 @@ RSpec.describe Database::PartitionManagementWorker do
describe '#perform' do
subject { described_class.new.perform }
let(:monitoring) { instance_double('PartitionMonitoring', report_metrics: nil) }
before do
allow(Gitlab::Database::Partitioning::PartitionMonitoring).to receive(:new).and_return(monitoring)
allow(Gitlab::Database::Partitioning).to receive(:sync_partitions)
allow(Gitlab::Database::Partitioning).to receive(:report_metrics)
end
it 'delegates to Partitioning' do
it 'syncs partitions' do
expect(Gitlab::Database::Partitioning).to receive(:sync_partitions)
subject
end
it 'reports partition metrics' do
expect(monitoring).to receive(:report_metrics)
expect(Gitlab::Database::Partitioning).to receive(:report_metrics)
subject
end