Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-09-06 09:11:02 +00:00
parent 578fc86533
commit aea64c367b
30 changed files with 240 additions and 224 deletions

View File

@ -27,14 +27,11 @@ code_quality:
variables:
SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific
SAST_EXCLUDED_PATHS: "qa, spec, doc, ee/spec, config/gitlab.yml.example, tmp" # GitLab-specific
SAST_EXCLUDED_ANALYZERS: bandit, flawfinder, phpcs-security-audit, pmd-apex, security-code-scan, spotbugs
SAST_EXCLUDED_ANALYZERS: bandit, flawfinder, phpcs-security-audit, pmd-apex, security-code-scan, spotbugs, eslint
brakeman-sast:
rules: !reference [".reports:rules:sast", rules]
eslint-sast:
rules: !reference [".reports:rules:sast", rules]
nodejs-scan-sast:
rules: !reference [".reports:rules:sast", rules]

View File

@ -1 +1 @@
50eff62c1b1f5730c7ca18597493b0b2926e72ca
0077763984dabc1056f34c698fc924cde5400133

View File

@ -14,6 +14,9 @@ module Types
argument :milestone_title, [GraphQL::Types::String],
required: false,
description: 'Milestone not applied to this issue.'
argument :author_username, GraphQL::Types::String,
required: false,
description: "Username of a user who didn't author the issue."
argument :assignee_usernames, [GraphQL::Types::String],
required: false,
description: 'Usernames of users not assigned to the issue.'

View File

@ -319,7 +319,6 @@ module Ci
scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }
scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) }
scope :with_pipeline_source, -> (source) { where(source: source)}
scope :outside_pipeline_family, ->(pipeline) do

View File

@ -223,33 +223,19 @@ class InternalId < ApplicationRecord
# If another process was faster in doing this, we'll end up with that record
# when we do the lookup after the insert.
def create_record
if Feature.enabled?(:use_insert_all_in_internal_id, default_enabled: :yaml)
scope[:project].save! if scope[:project] && !scope[:project].persisted?
scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted?
scope[:project].save! if scope[:project] && !scope[:project].persisted?
scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted?
attributes = {
project_id: scope[:project]&.id || scope[:project_id],
namespace_id: scope[:namespace]&.id || scope[:namespace_id],
usage: usage_value,
last_value: initial_value(subject, scope)
}
attributes = {
project_id: scope[:project]&.id || scope[:project_id],
namespace_id: scope[:namespace]&.id || scope[:namespace_id],
usage: usage_value,
last_value: initial_value(subject, scope)
}
InternalId.insert_all([attributes])
InternalId.insert(attributes)
lookup
else
begin
subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions
InternalId.create!(
**scope,
usage: usage_value,
last_value: initial_value(subject, scope)
)
end
rescue ActiveRecord::RecordNotUnique
lookup
end
end
lookup
end
end
@ -344,32 +330,21 @@ class InternalId < ApplicationRecord
end
def create_record!(subject, scope, usage, value)
if Feature.enabled?(:use_insert_all_in_internal_id, default_enabled: :yaml)
scope[:project].save! if scope[:project] && !scope[:project].persisted?
scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted?
scope[:project].save! if scope[:project] && !scope[:project].persisted?
scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted?
attributes = {
project_id: scope[:project]&.id || scope[:project_id],
namespace_id: scope[:namespace]&.id || scope[:namespace_id],
usage: usage_value,
last_value: value
}
attributes = {
project_id: scope[:project]&.id || scope[:project_id],
namespace_id: scope[:namespace]&.id || scope[:namespace_id],
usage: usage_value,
last_value: value
}
result = InternalId.insert_all([attributes])
result = InternalId.insert(attributes)
raise RecordAlreadyExists if result.empty?
raise RecordAlreadyExists if result.empty?
value
else
begin
subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions
internal_id = InternalId.create!(**scope, usage: usage, last_value: value)
internal_id.last_value
end
rescue ActiveRecord::RecordNotUnique
raise RecordAlreadyExists
end
end
value
end
def arel_table

View File

@ -1613,6 +1613,8 @@ class User < ApplicationRecord
true
end
# TODO Please check all callers and remove allow_cross_joins_across_databases,
# when https://gitlab.com/gitlab-org/gitlab/-/issues/336436 is done.
def ci_owned_runners
@ci_owned_runners ||= begin
project_runners = Ci::RunnerProject
@ -1629,6 +1631,12 @@ class User < ApplicationRecord
end
end
def owns_runner?(runner)
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do
ci_owned_runners.exists?(runner.id)
end
end
def notification_email_for(notification_group)
# Return group-specific email address if present, otherwise return global notification email address
notification_group&.notification_email_for(self) || notification_email

View File

@ -5,9 +5,9 @@ module Ci
with_options scope: :subject, score: 0
condition(:locked, scope: :subject) { @subject.locked? }
# rubocop: disable CodeReuse/ActiveRecord
condition(:owned_runner) { @user.ci_owned_runners.exists?(@subject.id) }
# rubocop: enable CodeReuse/ActiveRecord
condition(:owned_runner) do
@user.owns_runner?(@subject)
end
rule { anonymous }.prevent_all

View File

@ -19,7 +19,7 @@ class ExpirePipelineCacheWorker
# rubocop: disable CodeReuse/ActiveRecord
def perform(pipeline_id)
pipeline = Ci::Pipeline.eager_load_project.find_by(id: pipeline_id)
pipeline = Ci::Pipeline.find_by(id: pipeline_id)
return unless pipeline
Ci::ExpirePipelineCacheService.new.execute(pipeline)

View File

@ -1,8 +0,0 @@
---
name: cached_encoding_detection
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60128
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/328819
milestone: '13.12'
type: development
group: group::source code
default_enabled: false

View File

@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/334167
milestone: '14.1'
type: development
group: group::optimize
default_enabled: false
default_enabled: true

View File

@ -1,8 +0,0 @@
---
name: use_insert_all_in_internal_id
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68617
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339141
milestone: '14.3'
type: development
group: group::project management
default_enabled: false

View File

@ -1,20 +0,0 @@
---
data_category: operational
key_path: usage_activity_by_stage_monthly.secure.user_unique_users_all_secure_scanners
description:
product_section: sec
product_stage:
product_group: group::secure
product_category:
value_type: number
status: data_available
time_frame: 28d
data_source:
distribution:
- ce
tier:
- free
skip_validation: true
performance_indicator_type:
- smau
milestone: "<13.9"

View File

@ -1,13 +1,14 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_sast_monthly
description: ''
product_section: ''
product_stage: ''
product_group: ''
product_category: ''
description: Count of pipelines with implicit SAST runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: SAST
value_type: number
status: data_available
milestone: "<13.9"
time_frame: 28d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
@ -16,7 +17,8 @@ options:
- p_ci_templates_implicit_security_sast
distribution:
- ce
- ee
tier:
- free
skip_validation: true
milestone: "<13.9"
- premium
- ultimate

View File

@ -1,13 +1,14 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_secret_detection_monthly
description: ''
product_section: ''
product_stage: ''
product_group: ''
product_category: ''
description: Count of pipelines with implicit Secret Detection runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: secret_detection
value_type: number
status: data_available
milestone: "<13.9"
time_frame: 28d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
@ -16,7 +17,8 @@ options:
- p_ci_templates_implicit_security_secret_detection
distribution:
- ce
- ee
tier:
- free
skip_validation: true
milestone: "<13.9"
- premium
- ultimate

View File

@ -1,11 +1,11 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_security_sast_monthly
description: ''
product_section: ''
product_stage: ''
product_group: ''
product_category: ''
description: Count of pipelines with SAST runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: SAST
value_type: number
status: data_available
time_frame: 28d
@ -16,8 +16,10 @@ options:
- p_ci_templates_security_sast
distribution:
- ce
- ee
tier:
- free
skip_validation: true
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -1,11 +1,11 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_security_secret_detection_monthly
description: ''
product_section: ''
product_stage: ''
product_group: ''
product_category: ''
description: Count of pipelines with Secret Detection runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: secret_detection
value_type: number
status: data_available
time_frame: 28d
@ -16,8 +16,10 @@ options:
- p_ci_templates_security_secret_detection
distribution:
- ce
- ee
tier:
- free
skip_validation: true
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -0,0 +1,26 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_sast_weekly
description: Count of pipelines with implicit SAST runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: SAST
value_type: number
status: data_available
time_frame: 7d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
options:
events:
- p_ci_templates_implicit_security_sast
distribution:
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -0,0 +1,25 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_secret_detection_weekly
description: Count of pipelines with implicit Secret Detection runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: secret_detection
value_type: number
status: data_available
time_frame: 7d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
options:
events:
- p_ci_templates_implicit_security_secret_detection
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -0,0 +1,25 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_security_sast_weekly
description: Count of pipelines with SAST runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: SAST
value_type: number
status: data_available
time_frame: 7d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
options:
events:
- p_ci_templates_security_sast
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -0,0 +1,25 @@
---
data_category: optional
key_path: redis_hll_counters.ci_templates.p_ci_templates_security_secret_detection_weekly
description: Count of pipelines with Secret Detection runs
product_section: ops
product_stage: verify
product_group: "group::static analysis"
product_category: secret_detection
value_type: number
status: data_available
time_frame: 7d
data_source: redis_hll
instrumentation_class: RedisHLLMetric
options:
events:
- p_ci_templates_security_secret_detection
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
performance_indicator_type: []
milestone: "<13.9"

View File

@ -1,18 +0,0 @@
---
data_category: operational
key_path: usage_activity_by_stage.secure.user_unique_users_all_secure_scanners
description:
product_section: sec
product_stage:
product_group: group::secure
product_category:
value_type: number
status: data_available
time_frame: all
data_source:
distribution:
- ce
tier:
- free
skip_validation: true
milestone: "<13.9"

View File

@ -10,4 +10,5 @@ link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html
level: error
scope: raw
raw:
- '\*\*\(([Ff]ree|[Pp]remium|[Uu]ltimate)( [Ss](elf|ass))?\)\*\*'
- '(?!\*\*\((FREE|PREMIUM|ULTIMATE)( (SELF|SAAS))?\)\*\*)'
- '(?i)\*\*\((free|premium|ultimate)( (self|saas))?\)\*\*'

View File

@ -17548,6 +17548,7 @@ Represents an escalation rule.
| ---- | ---- | ----------- |
| <a id="negatedissuefilterinputassigneeid"></a>`assigneeId` | [`String`](#string) | ID of a user not assigned to the issues. |
| <a id="negatedissuefilterinputassigneeusernames"></a>`assigneeUsernames` | [`[String!]`](#string) | Usernames of users not assigned to the issue. |
| <a id="negatedissuefilterinputauthorusername"></a>`authorUsername` | [`String`](#string) | Username of a user who didn't author the issue. |
| <a id="negatedissuefilterinputepicid"></a>`epicId` | [`String`](#string) | ID of an epic not associated with the issues. |
| <a id="negatedissuefilterinputiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues to exclude. For example, `[1, 2]`. |
| <a id="negatedissuefilterinputiterationid"></a>`iterationId` | [`[ID!]`](#id) | List of iteration Global IDs not applied to the issue. |

View File

@ -40,15 +40,7 @@ module Gitlab
def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil)
return if data.nil?
if Feature.enabled?(:cached_encoding_detection, type: :development, default_enabled: :yaml)
return CharlockHolmes::EncodingDetector.new(limit).detect(data) unless cache_key.present?
Rails.cache.fetch([:detect_binary, CharlockHolmes::VERSION, cache_key], expires_in: 1.week) do
CharlockHolmes::EncodingDetector.new(limit).detect(data)
end
else
CharlockHolmes::EncodingDetector.new(limit).detect(data)
end
CharlockHolmes::EncodingDetector.new(limit).detect(data)
end
def detect_binary?(data, detect = nil)

View File

@ -6,6 +6,7 @@ RSpec.describe Resolvers::IssuesResolver do
include GraphqlHelpers
let_it_be(:current_user) { create(:user) }
let_it_be(:reporter) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
@ -27,6 +28,7 @@ RSpec.describe Resolvers::IssuesResolver do
context "with a project" do
before_all do
project.add_developer(current_user)
project.add_reporter(reporter)
create(:label_link, label: label1, target: issue1)
create(:label_link, label: label1, target: issue2)
create(:label_link, label: label2, target: issue2)
@ -235,6 +237,14 @@ RSpec.describe Resolvers::IssuesResolver do
it 'returns issues without the specified assignee_id' do
expect(resolve_issues(not: { assignee_id: [assignee.id] })).to contain_exactly(issue1)
end
context 'when filtering by negated author' do
let_it_be(:issue_by_reporter) { create(:issue, author: reporter, project: project, state: :opened) }
it 'returns issues without the specified author_username' do
expect(resolve_issues(not: { author_username: issue1.author.username })).to contain_exactly(issue_by_reporter)
end
end
end
describe 'sorting' do

View File

@ -241,7 +241,7 @@ RSpec.describe Gitlab::EncodingHelper do
let(:data) { binary_string }
let(:kwargs) { {} }
shared_examples 'detects encoding' do
context 'detects encoding' do
it { is_expected.to be_a(Hash) }
it 'correctly detects the binary' do
@ -264,33 +264,5 @@ RSpec.describe Gitlab::EncodingHelper do
end
end
end
context 'cached_encoding_detection is enabled' do
before do
stub_feature_flags(cached_encoding_detection: true)
end
it_behaves_like 'detects encoding'
context 'cache_key is provided' do
let(:kwargs) do
{ cache_key: %w(foo bar) }
end
it 'uses that cache_key to serve from the cache' do
expect(Rails.cache).to receive(:fetch).with([:detect_binary, CharlockHolmes::VERSION, %w(foo bar)], expires_in: 1.week).and_call_original
expect(subject[:type]).to eq(:binary)
end
end
end
context 'cached_encoding_detection is disabled' do
before do
stub_feature_flags(cached_encoding_detection: false)
end
it_behaves_like 'detects encoding'
end
end
end

View File

@ -243,21 +243,7 @@ RSpec.describe InternalId do
stub_feature_flags(generate_iids_without_explicit_locking: false)
end
context 'when the insert all feature flag is enabled' do
before do
stub_feature_flags(use_insert_all_in_internal_id: true)
end
it_behaves_like 'a monotonically increasing id generator'
end
context 'when the insert all feature flag is disabled' do
before do
stub_feature_flags(use_insert_all_in_internal_id: false)
end
it_behaves_like 'a monotonically increasing id generator'
end
it_behaves_like 'a monotonically increasing id generator'
end
context 'when the explicit locking feature flag is enabled' do
@ -265,21 +251,7 @@ RSpec.describe InternalId do
stub_feature_flags(generate_iids_without_explicit_locking: true)
end
context 'when the insert all feature flag is enabled' do
before do
stub_feature_flags(use_insert_all_in_internal_id: true)
end
it_behaves_like 'a monotonically increasing id generator'
end
context 'when the insert all feature flag is disabled' do
before do
stub_feature_flags(use_insert_all_in_internal_id: false)
end
it_behaves_like 'a monotonically increasing id generator'
end
it_behaves_like 'a monotonically increasing id generator'
end
describe '#increment_and_save!' do

View File

@ -3635,6 +3635,11 @@ RSpec.describe User do
it 'loads all the runners in the tree of groups' do
expect(user.ci_owned_runners).to contain_exactly(runner, group_runner)
end
it 'returns true for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(true)
expect(user.owns_runner?(group_runner)).to eq(true)
end
end
end
@ -3647,6 +3652,10 @@ RSpec.describe User do
it 'loads the runners in the group' do
expect(user.ci_owned_runners).to contain_exactly(group_runner)
end
it 'returns true for owns_runner?' do
expect(user.owns_runner?(group_runner)).to eq(true)
end
end
end
@ -3655,6 +3664,10 @@ RSpec.describe User do
it 'loads the runner belonging to the project' do
expect(user.ci_owned_runners).to contain_exactly(runner)
end
it 'returns true for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(true)
end
end
end
@ -3667,6 +3680,10 @@ RSpec.describe User do
it 'loads the runners of the project' do
expect(user.ci_owned_runners).to contain_exactly(project_runner)
end
it 'returns true for owns_runner?' do
expect(user.owns_runner?(project_runner)).to eq(true)
end
end
context 'when the user is a developer' do
@ -3677,6 +3694,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(project_runner)).to eq(false)
end
end
context 'when the user is a reporter' do
@ -3687,6 +3708,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(project_runner)).to eq(false)
end
end
context 'when the user is a guest' do
@ -3697,6 +3722,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(project_runner)).to eq(false)
end
end
end
@ -3709,6 +3738,10 @@ RSpec.describe User do
it 'does not load the runners of the group' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(false)
end
end
context 'when the user is a developer' do
@ -3719,6 +3752,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(false)
end
end
context 'when the user is a reporter' do
@ -3729,6 +3766,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(false)
end
end
context 'when the user is a guest' do
@ -3739,6 +3780,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(runner)).to eq(false)
end
end
end
@ -3746,6 +3791,10 @@ RSpec.describe User do
it 'does not load any runner' do
expect(user.ci_owned_runners).to be_empty
end
it 'returns false for owns_runner?' do
expect(user.owns_runner?(create(:ci_runner))).to eq(false)
end
end
context 'with runner in a personal project' do

View File

@ -386,7 +386,6 @@
- "./spec/workers/ci/drop_pipeline_worker_spec.rb"
- "./spec/workers/ci/initial_pipeline_process_worker_spec.rb"
- "./spec/workers/expire_job_cache_worker_spec.rb"
- "./spec/workers/expire_pipeline_cache_worker_spec.rb"
- "./spec/workers/new_merge_request_worker_spec.rb"
- "./spec/workers/pipeline_process_worker_spec.rb"
- "./spec/workers/pipeline_schedule_worker_spec.rb"

View File

@ -18,23 +18,6 @@ RSpec.describe ExpirePipelineCacheWorker do
subject.perform(pipeline.id)
end
it 'does not perform extra queries', :aggregate_failures do
recorder = ActiveRecord::QueryRecorder.new { subject.perform(pipeline.id) }
project_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "projects"')}
namespace_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "namespaces"')}
route_queries = recorder.data.values.flat_map {|v| v[:occurrences]}.select {|s| s.include?('FROM "routes"')}
# This worker is run 1 million times an hour, so we need to save as much
# queries as possible.
expect(recorder.count).to be <= 6
# These arises from #update_etag_cache
expect(project_queries.size).to eq(1)
expect(namespace_queries.size).to eq(1)
expect(route_queries.size).to eq(1)
end
it "doesn't do anything if the pipeline not exist" do
expect_any_instance_of(Ci::ExpirePipelineCacheService).not_to receive(:execute)
expect_any_instance_of(Gitlab::EtagCaching::Store).not_to receive(:touch)