kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2021-06-09 06:10:29 +00:00
parent bd02c91f73
commit a5628d3b6d
28 changed files with 317 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -42,6 +42,7 @@ eslint-report.html
/config/redis.cache.yml
/config/redis.queues.yml
/config/redis.shared_state.yml
/config/redis.trace_chunks.yml
/config/unicorn.rb
/config/puma.rb
/config/puma_actioncable.rb

2
.gitlab/CODEOWNERS
View file

@ -220,6 +220,8 @@ Dangerfile @gl-quality/eng-prod
/ee/app/policies/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team
/ee/app/policies/vulnerability*.rb @gitlab-org/secure/threat-insights-backend-team
/ee/lib/api/vulnerabilit*.rb @gitlab-org/secure/threat-insights-backend-team
/ee/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @gitlab-org/secure/threat-insights-backend-team
/ee/spec/lib/gitlab/ci/reports/security/vulnerability_reports_comparer_spec.rb @gitlab-org/secure/threat-insights-backend-team
/ee/spec/policies/vulnerabilities/ @gitlab-org/secure/threat-insights-backend-team
/ee/spec/policies/vulnerability*.rb @gitlab-org/secure/threat-insights-backend-team

2
app/controllers/admin/dashboard_controller.rb
View file

@ -15,7 +15,7 @@ class Admin::DashboardController < Admin::ApplicationController
@groups = Group.order_id_desc.with_route.limit(10)
@notices = Gitlab::ConfigChecker::PumaRuggedChecker.check
@notices += Gitlab::ConfigChecker::ExternalDatabaseChecker.check
@redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache].map(&:version).uniq
@redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache, Gitlab::Redis::TraceChunks].map(&:version).uniq
end
# rubocop: enable CodeReuse/ActiveRecord

1
app/controllers/health_controller.rb
View file

@ -15,6 +15,7 @@ class HealthController < ActionController::Base
Gitlab::HealthChecks::Redis::CacheCheck,
Gitlab::HealthChecks::Redis::QueuesCheck,
Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::Redis::TraceChunksCheck,
Gitlab::HealthChecks::GitalyCheck
].freeze

31
config/README.md
View file

@ -147,3 +147,34 @@ searched):
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
## redis.trace_chunks.yml
If configured, `redis.trace_chunks.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for clients of `::Gitlab::Redis::TraceChunks` which stores CI trace chunks.
Settings here can be overridden by the environment variable
`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` which provides
an alternate location for configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `trace_chunks` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable
2. URL from `redis.trace_chunks.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6383`
The order of precedence for all other configuration settings for `trace_chunks`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable
2. the configuration file `redis.trace_chunks.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`

8
config/feature_flags/development/load_balancing_atomic_replica.yml
View file

@ -1,8 +0,0 @@
---
name: load_balancing_atomic_replica
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49294
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/291193
milestone: '13.11'
type: development
group: group::database
default_enabled: true

1
config/initializers/7_redis.rb
View file

@ -8,3 +8,4 @@
Gitlab::Redis::Cache.with { nil }
Gitlab::Redis::Queues.with { nil }
Gitlab::Redis::SharedState.with { nil }
Gitlab::Redis::TraceChunks.with { nil }

38
config/redis.trace_chunks.yml.example Normal file
View file

@ -0,0 +1,38 @@
# If you change this file in a merge request, please also create
# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/13
#
# url: redis://localhost:6382
# sentinels:
# -
# host: localhost
# port: 26382 # point to sentinel, not to redis port
# -
# host: replica2
# port: 26382 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/13
#
# url: redis://localhost:6382
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.trace_chunks.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
##
# url: redis://master:6382
# sentinels:
# -
# host: replica1
# port: 26382 # point to sentinel, not to redis port
# -
# host: replica2
# port: 26382 # point to sentinel, not to redis port

27
doc/administration/reference_architectures/10k_users.md
View file

@ -1582,19 +1582,20 @@ To configure the Praefect nodes, on each one:
# server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
praefect['virtual_storages'] = {
'default' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
'nodes' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
}
}
}

27
doc/administration/reference_architectures/25k_users.md
View file

@ -1584,19 +1584,20 @@ To configure the Praefect nodes, on each one:
# server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
praefect['virtual_storages'] = {
'default' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
'nodes' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
}
}
}

27
doc/administration/reference_architectures/3k_users.md
View file

@ -1282,19 +1282,20 @@ To configure the Praefect nodes, on each one:
# server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
praefect['virtual_storages'] = {
'default' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
'nodes' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
}
}
}

27
doc/administration/reference_architectures/50k_users.md
View file

@ -1591,19 +1591,20 @@ To configure the Praefect nodes, on each one:
# server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
praefect['virtual_storages'] = {
'default' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
'nodes' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
}
}
}

27
doc/administration/reference_architectures/5k_users.md
View file

@ -1273,19 +1273,20 @@ To configure the Praefect nodes, on each one:
# server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
praefect['virtual_storages'] = {
'default' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
'nodes' => {
'gitaly-1' => {
'address' => 'tcp://10.6.0.91:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-2' => {
'address' => 'tcp://10.6.0.92:8075',
'token' => '<praefect_internal_token>'
},
'gitaly-3' => {
'address' => 'tcp://10.6.0.93:8075',
'token' => '<praefect_internal_token>'
},
}
}
}

6
lib/gitlab/checks/matching_merge_request.rb
View file

@ -28,11 +28,7 @@ module Gitlab
# report no matching merge requests. To avoid this, we check
# the write location to ensure the replica can make this query.
track_session_metrics do
if ::Feature.enabled?(:load_balancing_atomic_replica, @project, default_enabled: :yaml)
::Gitlab::Database::LoadBalancing::Sticking.select_valid_host(:project, @project.id)
else
::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(:project, @project.id)
end
::Gitlab::Database::LoadBalancing::Sticking.select_valid_host(:project, @project.id)
end
end

3
lib/gitlab/health_checks/redis/redis_check.rb
View file

@ -20,7 +20,8 @@ module Gitlab
def check
::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up
::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up &&
::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up
end
end
end

35
lib/gitlab/health_checks/redis/trace_chunks_check.rb Normal file
View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
module Gitlab
module HealthChecks
module Redis
class TraceChunksCheck
extend SimpleAbstractCheck
class << self
def check_up
check
end
private
def metric_prefix
'redis_trace_chunks_ping'
end
def successful?(result)
result == 'PONG'
end
# rubocop: disable CodeReuse/ActiveRecord
def check
catch_timeout 10.seconds do
Gitlab::Redis::TraceChunks.with(&:ping)
end
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
end
end

3
lib/gitlab/instrumentation/redis.rb
View file

@ -8,8 +8,9 @@ module Gitlab
Cache = Class.new(RedisBase).enable_redis_cluster_validation
Queues = Class.new(RedisBase)
SharedState = Class.new(RedisBase).enable_redis_cluster_validation
TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation
STORAGES = [ActionCable, Cache, Queues, SharedState].freeze
STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze
# Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze

4
lib/gitlab/redis/cache.rb
View file

@ -8,7 +8,9 @@ module Gitlab
private
def raw_config_hash
super || { url: 'redis://localhost:6380' }
config = super
config[:url] = 'redis://localhost:6380' if config[:url].blank?
config
end
end
end

5
lib/gitlab/redis/queues.rb
View file

@ -2,6 +2,7 @@
# We need this require for MailRoom
require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
require 'active_support/core_ext/object/blank'
module Gitlab
module Redis
@ -12,7 +13,9 @@ module Gitlab
private
def raw_config_hash
super || { url: 'redis://localhost:6381' }
config = super
config[:url] = 'redis://localhost:6381' if config[:url].blank?
config
end
end
end

4
lib/gitlab/redis/shared_state.rb
View file

@ -11,7 +11,9 @@ module Gitlab
private
def raw_config_hash
super || { url: 'redis://localhost:6382' }
config = super
config[:url] = 'redis://localhost:6382' if config[:url].blank?
config
end
end
end

12
lib/gitlab/redis/trace_chunks.rb Normal file
View file

@ -0,0 +1,12 @@
# frozen_string_literal: true
module Gitlab
module Redis
class TraceChunks < ::Gitlab::Redis::Wrapper
# The data we store on TraceChunks used to be stored on SharedState.
def self.config_fallback
SharedState
end
end
end
end

17
lib/gitlab/redis/wrapper.rb
View file

@ -64,8 +64,19 @@ module Gitlab
def config_file_name
[
# Instance specific config sources:
ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"],
config_file_path("redis.#{store_name.underscore}.yml"),
# The current Redis instance may have been split off from another one
# (e.g. TraceChunks was split off from SharedState). There are
# installations out there where the lowest priority config source
# (resque.yml) contains bogus values. In those cases, config_file_name
# should resolve to the instance we originated from (the
# "config_fallback") rather than resque.yml.
config_fallback&.config_file_name,
# Global config sources:
ENV['GITLAB_REDIS_CONFIG_FILE'],
config_file_path('resque.yml')
].compact.first
@ -75,6 +86,10 @@ module Gitlab
name.demodulize
end
def config_fallback
nil
end
def instrumentation_class
"::Gitlab::Instrumentation::Redis::#{store_name}".constantize
end
@ -133,6 +148,8 @@ module Gitlab
if config_data
config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
else
{ url: '' }
end
end

3
scripts/prepare_build.sh
View file

@ -42,6 +42,9 @@ sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
cp config/redis.shared_state.yml.example config/redis.shared_state.yml
sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
if [ "$SETUP_DB" != "false" ]; then
setup_db
elif getent hosts postgres; then

38
spec/lib/gitlab/checks/matching_merge_request_spec.rb
View file

@ -56,6 +56,9 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do
before do
expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).at_least(:once).and_return(true)
allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(all_caught_up)
expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_valid_host).with(:project, project.id).and_call_original
allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_caught_up_replicas).with(:project, project.id).and_return(all_caught_up)
end
shared_examples 'secondary that has caught up to a primary' do
@ -84,39 +87,12 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do
end
end
context 'with load_balancing_atomic_replica feature flag enabled' do
before do
stub_feature_flags(load_balancing_atomic_replica: true)
it_behaves_like 'secondary that has caught up to a primary'
expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_valid_host).with(:project, project.id).and_call_original
allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:select_caught_up_replicas).with(:project, project.id).and_return(all_caught_up)
end
context 'on secondary behind primary' do
let(:all_caught_up) { false }
it_behaves_like 'secondary that has caught up to a primary'
context 'on secondary behind primary' do
let(:all_caught_up) { false }
it_behaves_like 'secondary that is lagging primary'
end
end
context 'with load_balancing_atomic_replica feature flag disabled' do
before do
stub_feature_flags(load_balancing_atomic_replica: false)
expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:select_valid_host)
expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:unstick_or_continue_sticking).and_call_original
allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(all_caught_up)
end
it_behaves_like 'secondary that has caught up to a primary'
context 'on secondary behind primary' do
let(:all_caught_up) { false }
it_behaves_like 'secondary that is lagging primary'
end
it_behaves_like 'secondary that is lagging primary'
end
end
end

8
spec/lib/gitlab/health_checks/redis/trace_chunks_check_spec.rb Normal file
View file

@ -0,0 +1,8 @@
# frozen_string_literal: true
require 'spec_helper'
require_relative '../simple_check_shared'
RSpec.describe Gitlab::HealthChecks::Redis::TraceChunksCheck do
include_examples 'simple_check', 'redis_trace_chunks_ping', 'RedisTraceChunks', 'PONG'
end

3
spec/lib/gitlab/instrumentation/redis_spec.rb
View file

@ -68,7 +68,8 @@ RSpec.describe Gitlab::Instrumentation::Redis do
.to contain_exactly(details_row.merge(storage: 'ActionCable'),
details_row.merge(storage: 'Cache'),
details_row.merge(storage: 'Queues'),
details_row.merge(storage: 'SharedState'))
details_row.merge(storage: 'SharedState'),
details_row.merge(storage: 'TraceChunks'))
end
end
end

55
spec/lib/gitlab/redis/trace_chunks_spec.rb Normal file
View file

@ -0,0 +1,55 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Redis::TraceChunks do
let(:instance_specific_config_file) { "config/redis.trace_chunks.yml" }
let(:environment_config_file_name) { "GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE" }
let(:shared_state_config_file) { nil }
before do
allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(shared_state_config_file)
end
include_examples "redis_shared_examples"
describe '.config_file_name' do
subject { described_class.config_file_name }
let(:rails_root) { Dir.mktmpdir('redis_shared_examples') }
before do
# Undo top-level stub of config_file_name because we are testing that method now.
allow(described_class).to receive(:config_file_name).and_call_original
allow(described_class).to receive(:rails_root).and_return(rails_root)
FileUtils.mkdir_p(File.join(rails_root, 'config'))
end
after do
FileUtils.rm_rf(rails_root)
end
context 'when there is only a resque.yml' do
before do
FileUtils.touch(File.join(rails_root, 'config/resque.yml'))
end
it { expect(subject).to eq("#{rails_root}/config/resque.yml") }
context 'and there is a global env override' do
before do
stub_env('GITLAB_REDIS_CONFIG_FILE', 'global override')
end
it { expect(subject).to eq('global override') }
context 'and SharedState has a different config file' do
let(:shared_state_config_file) { 'shared state config file' }
it { expect(subject).to eq('shared state config file') }
end
end
end
end
end

20
spec/support/redis/redis_shared_examples.rb
View file

@ -16,6 +16,7 @@ RSpec.shared_examples "redis_shared_examples" do
let(:sentinel_port) { 26379 }
let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"}
let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"}
let(:rails_root) { Dir.mktmpdir('redis_shared_examples') }
before do
allow(described_class).to receive(:config_file_name).and_return(Rails.root.join(config_file_name).to_s)
@ -29,8 +30,6 @@ RSpec.shared_examples "redis_shared_examples" do
describe '.config_file_name' do
subject { described_class.config_file_name }
let(:rails_root) { Dir.mktmpdir('redis_shared_examples') }
before do
# Undo top-level stub of config_file_name because we are testing that method now.
allow(described_class).to receive(:config_file_name).and_call_original
@ -237,6 +236,23 @@ RSpec.shared_examples "redis_shared_examples" do
described_class.with { |_redis_shared_example| true }
end
end
context 'when there is no config at all' do
before do
# Undo top-level stub of config_file_name because we are testing that method now.
allow(described_class).to receive(:config_file_name).and_call_original
allow(described_class).to receive(:rails_root).and_return(rails_root)
end
after do
FileUtils.rm_rf(rails_root)
end
it 'can run an empty block' do
expect { described_class.with { nil } }.not_to raise_error
end
end
end
describe '#sentinels' do