kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-12-14 03:13:32 +00:00
parent 1d54f384d5
commit be4229fb3a
25 changed files with 310 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.rubocop_todo/rails/save_bang.yml
View File

@ -19,13 +19,6 @@ Rails/SaveBang:
- ee/spec/models/elasticsearch_indexed_namespace_spec.rb
- ee/spec/models/gitlab_subscription_spec.rb
- ee/spec/models/issue_spec.rb
- ee/spec/models/label_note_spec.rb
- ee/spec/models/lfs_object_spec.rb
- ee/spec/models/license_spec.rb
- ee/spec/models/merge_request_spec.rb
- ee/spec/models/merge_train_spec.rb
- ee/spec/models/project_ci_cd_setting_spec.rb
- ee/spec/models/project_spec.rb
- ee/spec/models/protected_environment_spec.rb
- ee/spec/models/repository_spec.rb
- ee/spec/models/scim_identity_spec.rb

2
app/serializers/deployment_entity.rb
View File

@ -22,6 +22,8 @@ class DeploymentEntity < Grape::Entity
expose :deployed_at
expose :tag
expose :last?
expose :last?, as: :is_last
expose :deployed_by, as: :user, using: UserEntity
expose :deployable, if: -> (deployment) { deployment.deployable.present? } do |deployment, opts|

2
app/serializers/deployment_serializer.rb
View File

@ -4,7 +4,7 @@ class DeploymentSerializer < BaseSerializer
entity DeploymentEntity
def represent_concise(resource, opts = {})
opts[:only] = [:iid, :id, :sha, :created_at, :deployed_at, :tag, :last?, :id, ref: [:name]]
opts[:only] = [:iid, :id, :sha, :created_at, :deployed_at, :tag, :last?, :is_last, :id, ref: [:name]]
represent(resource, opts)
end
end

2
app/views/shared/integrations/overrides.html.haml
View File

@ -1,4 +1,4 @@
- add_to_breadcrumbs _('Integrations'), scoped_integrations_path(project: project, group: group)
- add_to_breadcrumbs _('Integrations'), scoped_integrations_path(project: @project, group: @group)
- breadcrumb_title @integration.title
- page_title @integration.title, _('Integrations')
- @content_class = 'limit-container-width' unless fluid_layout

2
config/feature_flags/development/improved_container_scan_matching.yml
View File

@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344534
milestone: '14.6'
type: development
group: group::container security
default_enabled: false
default_enabled: true

22
db/post_migrate/20211023102243_schedule_delete_invalid_epic_issues.rb Normal file
View File

@ -0,0 +1,22 @@
# frozen_string_literal: true
class ScheduleDeleteInvalidEpicIssues < Gitlab::Database::Migration[1.0]
MIGRATION = 'DeleteInvalidEpicIssues'
DELAY_INTERVAL = 2.minutes
BATCH_SIZE = 10_000
disable_ddl_transaction!
def up
queue_background_migration_jobs_by_range_at_intervals(
define_batchable_model('epics'),
MIGRATION,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true
)
end
def down
end
end

25
db/post_migrate/20211118194239_drop_invalid_remediations.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
class DropInvalidRemediations < Gitlab::Database::Migration[1.0]
BATCH_SIZE = 3_000
DELAY_INTERVAL = 3.minutes
MIGRATION_NAME = 'DropInvalidRemediations'
DAY_PRIOR_TO_BUG_INTRODUCTION = DateTime.new(2021, 8, 1, 0, 0, 0)
disable_ddl_transaction!
def up
return unless Gitlab.ee?
relation = Gitlab::BackgroundMigration::DropInvalidRemediations::FindingRemediation.where("created_at > ?", DAY_PRIOR_TO_BUG_INTRODUCTION)
queue_background_migration_jobs_by_range_at_intervals(relation,
MIGRATION_NAME,
DELAY_INTERVAL,
batch_size: BATCH_SIZE,
track_jobs: true)
end
def down
# no-op
end
end

1
db/schema_migrations/20211023102243 Normal file
View File

@ -0,0 +1 @@
f5039be0bd028dab4f2623fe9997a95d50bd9020ffd8b92074418024cda39b6a

1
db/schema_migrations/20211118194239 Normal file
View File

@ -0,0 +1 @@
04a4b10085bae2006ac78600b3cc410d130f9ac6944103c7bd85f71e060d4a67

13
lib/gitlab/background_migration/delete_invalid_epic_issues.rb Normal file
View File

@ -0,0 +1,13 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class DeleteInvalidEpicIssues
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::DeleteInvalidEpicIssues.prepend_mod_with('Gitlab::BackgroundMigration::DeleteInvalidEpicIssues')

14
lib/gitlab/background_migration/drop_invalid_remediations.rb Normal file
View File

@ -0,0 +1,14 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class DropInvalidRemediations
def perform(start_id, stop_id)
end
end
# rubocop: enable Style/Documentation
end
end
Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations')

4
lib/gitlab/database/async_indexes/index_creator.rb
View File

@ -47,6 +47,10 @@ module Gitlab
TIMEOUT_PER_ACTION
end
def lease_key
[super, async_index.connection_db_config.name].join('/')
end
def set_statement_timeout
connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
yield

20
lib/gitlab/database/reindexing.rb
View File

@ -15,6 +15,26 @@ module Gitlab
# on e.g. vacuum.
REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30
def self.enabled?
Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml)
end
def self.invoke(database = nil)
Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name|
next if database && database.to_s != connection_name.to_s
Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
# Hack: Before we do actual reindexing work, create async indexes
Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
automatic_reindexing
end
rescue StandardError => e
Gitlab::AppLogger.error(e)
raise
end
# Performs automatic reindexing for a limited number of indexes per call
# 1. Consume from the explicit reindexing queue
# 2. Apply bloat heuristic to find most bloated indexes and reindex those

4
lib/gitlab/database/reindexing/coordinator.rb
View File

@ -53,6 +53,10 @@ module Gitlab
def lease_timeout
TIMEOUT_PER_ACTION
end
def lease_key
[super, index.connection_db_config.name].join('/')
end
end
end
end

4
lib/gitlab/database/shared_model.rb
View File

@ -39,6 +39,10 @@ module Gitlab
Thread.current[:overriding_connection] = connection
end
end
def connection_db_config
self.class.connection_db_config
end
end
end
end

26
lib/tasks/gitlab/db.rake
View File

@ -175,24 +175,30 @@ namespace :gitlab do
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
desc 'execute reindexing without downtime to eliminate bloat'
desc "Reindex database without downtime to eliminate bloat"
task reindex: :environment do
unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml)
unless Gitlab::Database::Reindexing.enabled?
puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
exit
end
Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name|
Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
Gitlab::Database::Reindexing.invoke
end
# Hack: Before we do actual reindexing work, create async indexes
Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
namespace :reindex do
databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
Gitlab::Database::Reindexing.automatic_reindexing
ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
desc "Reindex #{database_name} database without downtime to eliminate bloat"
task database_name => :environment do
unless Gitlab::Database::Reindexing.enabled?
puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
exit
end
Gitlab::Database::Reindexing.invoke(database_name)
end
end
rescue StandardError => e
Gitlab::AppLogger.error(e)
raise
end
desc 'Enqueue an index for reindexing'

36
spec/controllers/admin/integrations_controller_spec.rb
View File

@ -105,40 +105,4 @@ RSpec.describe Admin::IntegrationsController do
.and change { Integrations::Jira.inherit_from_id(integration.id).count }.by(-1)
end
end
describe '#overrides' do
let_it_be(:instance_integration) { create(:bugzilla_integration, :instance) }
let_it_be(:non_overridden_integration) { create(:bugzilla_integration, inherit_from_id: instance_integration.id) }
let_it_be(:overridden_integration) { create(:bugzilla_integration) }
let_it_be(:overridden_other_integration) { create(:confluence_integration) }
subject do
get :overrides, params: { id: instance_integration.class.to_param }, format: format
end
context 'when format is JSON' do
let(:format) { :json }
include_context 'JSON response'
it 'returns projects with overrides', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response).to include_pagination_headers
expect(json_response).to contain_exactly(a_hash_including('full_name' => overridden_integration.project.full_name))
end
end
context 'when format is HTML' do
let(:format) { :html }
it 'renders template' do
subject
expect(response).to render_template 'shared/integrations/overrides'
expect(assigns(:integration)).to eq(instance_integration)
end
end
end
end

2
spec/fixtures/api/schemas/deployment.json vendored
View File

@ -7,6 +7,7 @@
"iid",
"tag",
"last?",
"is_last",
"ref",
"id"
],
@ -16,6 +17,7 @@
"id": { "type": "integer" },
"iid": { "type": "integer" },
"last?": { "type": "boolean" },
"is_last": { "type": "boolean" },
"ref": {
"type": "object",
"required": [

17
spec/lib/gitlab/database/async_indexes/index_creator_spec.rb
View File

@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do
include ExclusiveLeaseHelpers
describe '#perform' do
subject { described_class.new(async_index) }
@ -10,7 +12,18 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do
let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex }
let(:connection) { ApplicationRecord.connection }
let(:model) { Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] }
let(:connection) { model.connection }
let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) }
let(:lease_key) { "gitlab/database/async_indexes/index_creator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" }
let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION }
around do |example|
Gitlab::Database::SharedModel.using_connection(connection) do
example.run
end
end
context 'when the index already exists' do
before do
@ -40,7 +53,7 @@ RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do
end
it 'skips logic if not able to acquire exclusive lease' do
expect(subject).to receive(:try_obtain_lease).and_return(false)
expect(lease).to receive(:try_obtain).ordered.and_return(false)
expect(connection).not_to receive(:execute).with(/CREATE INDEX/)
expect(async_index).not_to receive(:destroy)

10
spec/lib/gitlab/database/reindexing/coordinator_spec.rb
View File

@ -15,10 +15,18 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do
let(:action) { create(:reindex_action, index: index) }
let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) }
let(:lease_key) { 'gitlab/database/reindexing/coordinator' }
let(:lease_key) { "gitlab/database/reindexing/coordinator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" }
let(:lease_timeout) { 1.day }
let(:uuid) { 'uuid' }
around do |example|
model = Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME]
Gitlab::Database::SharedModel.using_connection(model.connection) do
example.run
end
end
before do
swapout_view_for_table(:postgres_indexes)

74
spec/lib/gitlab/database/reindexing_spec.rb
View File

@ -6,6 +6,63 @@ RSpec.describe Gitlab::Database::Reindexing do
include ExclusiveLeaseHelpers
include Database::DatabaseHelpers
describe '.invoke' do
let(:databases) { Gitlab::Database.database_base_models }
let(:databases_count) { databases.count }
it 'cleans up any leftover indexes' do
expect(described_class).to receive(:cleanup_leftovers!).exactly(databases_count).times
described_class.invoke
end
context 'when there is an error raised' do
it 'logs and re-raise' do
expect(described_class).to receive(:automatic_reindexing).and_raise('Unexpected!')
expect(Gitlab::AppLogger).to receive(:error)
expect { described_class.invoke }.to raise_error('Unexpected!')
end
end
context 'when async index creation is enabled' do
it 'executes async index creation prior to any reindexing actions' do
stub_feature_flags(database_async_index_creation: true)
expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered.exactly(databases_count).times
expect(described_class).to receive(:automatic_reindexing).ordered.exactly(databases_count).times
described_class.invoke
end
end
context 'when async index creation is disabled' do
it 'does not execute async index creation' do
stub_feature_flags(database_async_index_creation: false)
expect(Gitlab::Database::AsyncIndexes).not_to receive(:create_pending_indexes!)
described_class.invoke
end
end
context 'calls automatic reindexing' do
it 'uses all candidate indexes' do
expect(described_class).to receive(:automatic_reindexing).exactly(databases_count).times
described_class.invoke
end
context 'when explicit database is given' do
it 'skips other databases' do
expect(described_class).to receive(:automatic_reindexing).once
described_class.invoke(Gitlab::Database::PRIMARY_DATABASE_NAME)
end
end
end
end
describe '.automatic_reindexing' do
subject { described_class.automatic_reindexing(maximum_records: limit) }
@ -133,10 +190,19 @@ RSpec.describe Gitlab::Database::Reindexing do
end
describe '.cleanup_leftovers!' do
subject { described_class.cleanup_leftovers! }
subject(:cleanup_leftovers) { described_class.cleanup_leftovers! }
let(:model) { Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME] }
let(:connection) { model.connection }
around do |example|
Gitlab::Database::SharedModel.using_connection(connection) do
example.run
end
end
before do
ApplicationRecord.connection.execute(<<~SQL)
connection.execute(<<~SQL)
CREATE INDEX foobar_ccnew ON users (id);
CREATE INDEX foobar_ccnew1 ON users (id);
SQL
@ -150,11 +216,11 @@ RSpec.describe Gitlab::Database::Reindexing do
expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"foobar_ccnew1\"")
expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout")
subject
cleanup_leftovers
end
def expect_query(sql)
expect(ApplicationRecord.connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql|
expect(connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql|
method.call(sql.sub(/CONCURRENTLY/, ''))
end
end

12
spec/lib/gitlab/database/shared_model_spec.rb
View File

@ -84,4 +84,16 @@ RSpec.describe Gitlab::Database::SharedModel do
expect(described_class.connection).to be(original_connection)
end
end
describe '#connection_db_config' do
it 'returns the class connection_db_config' do
shared_model_class = Class.new(described_class) do
self.table_name = 'postgres_async_indexes'
end
shared_model = shared_model_class.new
expect(shared_model.connection_db_config). to eq(described_class.connection_db_config)
end
end
end

50
spec/requests/admin/integrations_controller_spec.rb Normal file
View File

@ -0,0 +1,50 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Admin::IntegrationsController, :enable_admin_mode do
let_it_be(:admin) { create(:admin) }
before do
sign_in(admin)
end
describe 'GET #overrides' do
let_it_be(:integration) { create(:jira_integration, :instance) }
let_it_be(:overridden_integration) { create(:jira_integration) }
let_it_be(:overridden_other_integration) { create(:confluence_integration) }
let(:overrides_path) { overrides_admin_application_settings_integration_path(integration, format: format) }
context 'format html' do
let(:format) { :html }
it 'renders' do
get overrides_path
expect(response).to have_gitlab_http_status(:ok)
expect(response).to render_template('shared/integrations/overrides')
end
end
context 'format json' do
let(:format) { :json }
let(:project) { overridden_integration.project }
it 'returns the project overrides data' do
get overrides_path
expect(response).to have_gitlab_http_status(:ok)
expect(response).to include_pagination_headers
expect(json_response).to contain_exactly(
{
'avatar_url' => project.avatar_url,
'full_name' => project.full_name,
'name' => project.name,
'full_path' => project_path(project)
}
)
end
end
end
end

4
spec/serializers/deployment_entity_spec.rb
View File

@ -42,6 +42,10 @@ RSpec.describe DeploymentEntity do
expect(subject).to include(:deployed_at)
end
it 'exposes last? as is_last' do
expect(subject).to include(:is_last)
end
context 'when deployable is nil' do
let(:entity) { described_class.new(deployment, request: request, deployment_details: false) }
let(:deployment) { create(:deployment, deployable: nil, project: project) }

43
spec/tasks/gitlab/db_rake_spec.rb
View File

@ -203,43 +203,38 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
end
describe 'reindex' do
let(:reindex) { double('reindex') }
let(:indexes) { double('indexes') }
let(:databases) { Gitlab::Database.database_base_models }
let(:databases_count) { databases.count }
it 'cleans up any leftover indexes' do
expect(Gitlab::Database::Reindexing).to receive(:cleanup_leftovers!).exactly(databases_count).times
it 'delegates to Gitlab::Database::Reindexing' do
expect(Gitlab::Database::Reindexing).to receive(:invoke)
run_rake_task('gitlab:db:reindex')
end
context 'when async index creation is enabled' do
it 'executes async index creation prior to any reindexing actions' do
stub_feature_flags(database_async_index_creation: true)
expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered.exactly(databases_count).times
expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).ordered.exactly(databases_count).times
context 'when reindexing is not enabled' do
it 'is a no-op' do
expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
expect(Gitlab::Database::Reindexing).not_to receive(:invoke)
run_rake_task('gitlab:db:reindex')
end
end
end
context 'when async index creation is disabled' do
it 'does not execute async index creation' do
stub_feature_flags(database_async_index_creation: false)
databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
describe "reindex:#{database_name}" do
it 'delegates to Gitlab::Database::Reindexing' do
expect(Gitlab::Database::Reindexing).to receive(:invoke).with(database_name)
expect(Gitlab::Database::AsyncIndexes).not_to receive(:create_pending_indexes!)
run_rake_task('gitlab:db:reindex')
run_rake_task("gitlab:db:reindex:#{database_name}")
end
end
context 'calls automatic reindexing' do
it 'uses all candidate indexes' do
expect(Gitlab::Database::Reindexing).to receive(:automatic_reindexing).exactly(databases_count).times
context 'when reindexing is not enabled' do
it 'is a no-op' do
expect(Gitlab::Database::Reindexing).to receive(:enabled?).and_return(false)
expect(Gitlab::Database::Reindexing).not_to receive(:invoke).with(database_name)
run_rake_task('gitlab:db:reindex')
run_rake_task("gitlab:db:reindex:#{database_name}")
end
end
end
end