kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-06-29 06:09:45 +00:00
parent 7d4e6bdbfb
commit 06bd645177
14 changed files with 352 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
config/initializers/enumerator_next_patch.rb Normal file
View File

@ -0,0 +1,44 @@
# frozen_string_literal: true
# Monkey patch of Enumerator#next to get better stack traces
# when an error is raised from within a Fiber.
# https://bugs.ruby-lang.org/issues/16829
module EnumeratorNextPatch
%w(next next_values peek peek_values).each do |name|
define_method(name) do |*args|
gitlab_patch_backtrace_marker { super(*args) }
rescue Exception => err # rubocop: disable Lint/RescueException
err.set_backtrace(err.backtrace + caller) unless
has_gitlab_patch_backtrace_marker?(err.backtrace) && backtrace_matches_caller?(err.backtrace)
raise
end
end
private
def gitlab_patch_backtrace_marker
yield
end
# This function tells us whether the exception was generated by #next itself or by something in
# the Fiber that it invokes. If it's generated by #next, then the backtrace will have
# #gitlab_patch_backtrace_marker as the third item down the trace (since
# #gitlab_patch_backtrace_marker calls a block, which in turn calls #next.) If it's generated
# by the Fiber that #next invokes, then it won't contain this marker.
def has_gitlab_patch_backtrace_marker?(backtrace)
match = %r(^(.*):[0-9]+:in `gitlab_patch_backtrace_marker'$).match(backtrace[2])
!!match && match[1] == __FILE__
end
# This function makes sure that the rest of the stack trace matches in order to avoid missing
# an exception that was generated by calling #next on another Enumerator inside the Fiber.
# This might miss some *very* contrived scenarios involving recursion, but exceptions don't
# provide Fiber information, so it's the best we can do.
def backtrace_matches_caller?(backtrace)
backtrace[3..] == caller[1..]
end
end
Enumerator.prepend(EnumeratorNextPatch)

9
db/migrate/20220523162734_add_vulnerability_reads_casted_cluster_agent_column.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddVulnerabilityReadsCastedClusterAgentColumn < Gitlab::Database::Migration[2.0]
enable_lock_retries!
def change
add_column :vulnerability_reads, :casted_cluster_agent_id, :bigint
end
end

81
db/migrate/20220523163734_update_vulnerability_reads_trigger_functions.rb Normal file
View File

@ -0,0 +1,81 @@
# frozen_string_literal: true
class UpdateVulnerabilityReadsTriggerFunctions < Gitlab::Database::Migration[2.0]
AGENT_ID_VALUE = "NEW.location->'kubernetes_resource'->>'agent_id'"
CASTED_AGENT_ID_VALUE = "CAST(#{AGENT_ID_VALUE} AS bigint)"
def up
update_insert_or_update_vulnerability_reads_function(with_casted_cluster_agent_id: true)
update_update_location_from_vulnerability_occurrences_function(with_casted_cluster_agent_id: true)
end
def down
update_insert_or_update_vulnerability_reads_function(with_casted_cluster_agent_id: false)
update_update_location_from_vulnerability_occurrences_function(with_casted_cluster_agent_id: false)
end
private
def update_insert_or_update_vulnerability_reads_function(with_casted_cluster_agent_id: false)
insert_fields = with_casted_cluster_agent_id ? 'cluster_agent_id, casted_cluster_agent_id' : 'cluster_agent_id'
insert_values = with_casted_cluster_agent_id ? [AGENT_ID_VALUE, CASTED_AGENT_ID_VALUE].join(', ') : AGENT_ID_VALUE
execute(<<~SQL)
CREATE OR REPLACE FUNCTION insert_or_update_vulnerability_reads()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
severity smallint;
state smallint;
report_type smallint;
resolved_on_default_branch boolean;
BEGIN
IF (NEW.vulnerability_id IS NULL AND (TG_OP = 'INSERT' OR TG_OP = 'UPDATE')) THEN
RETURN NULL;
END IF;
IF (TG_OP = 'UPDATE' AND OLD.vulnerability_id IS NOT NULL AND NEW.vulnerability_id IS NOT NULL) THEN
RETURN NULL;
END IF;
SELECT
vulnerabilities.severity, vulnerabilities.state, vulnerabilities.report_type, vulnerabilities.resolved_on_default_branch
INTO
severity, state, report_type, resolved_on_default_branch
FROM
vulnerabilities
WHERE
vulnerabilities.id = NEW.vulnerability_id;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, #{insert_fields})
VALUES (NEW.vulnerability_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', #{insert_values})
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
$$;
SQL
end
def update_update_location_from_vulnerability_occurrences_function(with_casted_cluster_agent_id: false)
execute(<<~SQL)
CREATE OR REPLACE FUNCTION update_location_from_vulnerability_occurrences()
RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
UPDATE
vulnerability_reads
SET
location_image = NEW.location->>'image',
#{with_casted_cluster_agent_id ? "casted_cluster_agent_id = #{CASTED_AGENT_ID_VALUE}," : ''}
cluster_agent_id = #{AGENT_ID_VALUE}
WHERE
vulnerability_id = NEW.vulnerability_id;
RETURN NULL;
END
$$;
SQL
end
end

16
db/post_migrate/20220523164734_add_foreign_key_to_vulnerability_reads_casted_cluster_agent_id.rb Normal file
View File

@ -0,0 +1,16 @@
# frozen_string_literal: true
class AddForeignKeyToVulnerabilityReadsCastedClusterAgentId < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
def up
add_concurrent_foreign_key :vulnerability_reads, :cluster_agents,
column: :casted_cluster_agent_id, on_delete: :nullify
end
def down
with_lock_retries do
remove_foreign_key :vulnerability_reads, :cluster_agents, column: :casted_cluster_agent_id
end
end
end

15
db/post_migrate/20220523165734_add_index_to_vulnerability_reads_casted_cluster_agent_id.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
class AddIndexToVulnerabilityReadsCastedClusterAgentId < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
INDEX_NAME = 'index_cis_vulnerability_reads_on_cluster_agent_id'
def up
add_concurrent_index :vulnerability_reads, :casted_cluster_agent_id, name: INDEX_NAME, where: 'report_type = 7'
end
def down
remove_concurrent_index_by_name :vulnerability_reads, INDEX_NAME
end
end

1
db/schema_migrations/20220523162734 Normal file
View File

@ -0,0 +1 @@
af6f954426b714649a3b19e80a20cf99475cdc8496c23add8032cda27072d31d

1
db/schema_migrations/20220523163734 Normal file
View File

@ -0,0 +1 @@
02821039feda457c1fb61dc6ff62756752c0c1c0ad01e12ecf28c265462529d4

1
db/schema_migrations/20220523164734 Normal file
View File

@ -0,0 +1 @@
ef078bbcf8415a7bb49ed919739005d22c21199da1a0e5e5c0971d2a8e1b2a40

1
db/schema_migrations/20220523165734 Normal file
View File

@ -0,0 +1 @@
89a03d69c44ed95133446275bb9b39dfe91ad3022cefdfa438ea3c96ab4f8e69

11
db/structure.sql
View File

@ -71,8 +71,8 @@ BEGIN
WHERE
vulnerabilities.id = NEW.vulnerability_id;
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id)
VALUES (NEW.vulnerability_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id')
INSERT INTO vulnerability_reads (vulnerability_id, project_id, scanner_id, report_type, severity, state, resolved_on_default_branch, uuid, location_image, cluster_agent_id, casted_cluster_agent_id)
VALUES (NEW.vulnerability_id, NEW.project_id, NEW.scanner_id, report_type, severity, state, resolved_on_default_branch, NEW.uuid::uuid, NEW.location->>'image', NEW.location->'kubernetes_resource'->>'agent_id', CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint))
ON CONFLICT(vulnerability_id) DO NOTHING;
RETURN NULL;
END
@ -206,6 +206,7 @@ UPDATE
vulnerability_reads
SET
location_image = NEW.location->>'image',
casted_cluster_agent_id = CAST(NEW.location->'kubernetes_resource'->>'agent_id' AS bigint),
cluster_agent_id = NEW.location->'kubernetes_resource'->>'agent_id'
WHERE
vulnerability_id = NEW.vulnerability_id;
@ -22211,6 +22212,7 @@ CREATE TABLE vulnerability_reads (
uuid uuid NOT NULL,
location_image text,
cluster_agent_id text,
casted_cluster_agent_id bigint,
CONSTRAINT check_380451bdbe CHECK ((char_length(location_image) <= 2048)),
CONSTRAINT check_a105eb825a CHECK ((char_length(cluster_agent_id) <= 10))
);
@ -27655,6 +27657,8 @@ CREATE UNIQUE INDEX index_ci_variables_on_project_id_and_key_and_environment_sco
CREATE INDEX index_cicd_settings_on_namespace_id_where_stale_pruning_enabled ON namespace_ci_cd_settings USING btree (namespace_id) WHERE (allow_stale_runner_pruning = true);
CREATE INDEX index_cis_vulnerability_reads_on_cluster_agent_id ON vulnerability_reads USING btree (casted_cluster_agent_id) WHERE (report_type = 7);
CREATE INDEX index_cluster_agent_tokens_on_agent_id_status_last_used_at ON cluster_agent_tokens USING btree (agent_id, status, last_used_at DESC NULLS LAST);
CREATE INDEX index_cluster_agent_tokens_on_created_by_user_id ON cluster_agent_tokens USING btree (created_by_user_id);
@ -32030,6 +32034,9 @@ ALTER TABLE ONLY merge_requests
ALTER TABLE ONLY merge_request_metrics
ADD CONSTRAINT fk_ae440388cc FOREIGN KEY (latest_closed_by_id) REFERENCES users(id) ON DELETE SET NULL;
ALTER TABLE ONLY vulnerability_reads
ADD CONSTRAINT fk_aee839e611 FOREIGN KEY (casted_cluster_agent_id) REFERENCES cluster_agents(id) ON DELETE SET NULL;
ALTER TABLE ONLY dast_profile_schedules
ADD CONSTRAINT fk_aef03d62e5 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL;

2
doc/api/graphql/reference/index.md
View File

@ -16719,8 +16719,10 @@ Represents the scan result policy.
| ---- | ---- | ----------- |
| <a id="scanresultpolicydescription"></a>`description` | [`String!`](#string) | Description of the policy. |
| <a id="scanresultpolicyenabled"></a>`enabled` | [`Boolean!`](#boolean) | Indicates whether this policy is enabled. |
| <a id="scanresultpolicygroupapprovers"></a>`groupApprovers` | [`[Group!]`](#group) | Approvers of the group type. |
| <a id="scanresultpolicyname"></a>`name` | [`String!`](#string) | Name of the policy. |
| <a id="scanresultpolicyupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp of when the policy YAML was last updated. |
| <a id="scanresultpolicyuserapprovers"></a>`userApprovers` | [`[UserCore!]`](#usercore) | Approvers of the user type. |
| <a id="scanresultpolicyyaml"></a>`yaml` | [`String!`](#string) | YAML definition of the policy. |
### `ScannedResource`

2
qa/qa/runtime/browser.rb
View File

@ -16,7 +16,7 @@ module QA
NotRespondingError = Class.new(RuntimeError)
CAPYBARA_MAX_WAIT_TIME = 10
CAPYBARA_MAX_WAIT_TIME = Env.max_capybara_wait_time
def self.blank_page?
['', 'about:blank', 'data:,'].include?(Capybara.current_session.driver.browser.current_url)

4
qa/qa/runtime/env.rb
View File

@ -461,6 +461,10 @@ module QA
enabled?(ENV['QA_SKIP_SMOKE_RELIABLE'], default: false)
end
def max_capybara_wait_time
ENV.fetch('MAX_CAPYBARA_WAIT_TIME', 10).to_i
end
private
def remote_grid_credentials

167
spec/initializers/enumerator_next_patch_spec.rb Normal file
View File

@ -0,0 +1,167 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Enumerator#next patch fix' do
describe 'Enumerator' do
RSpec::Matchers.define :contain_unique_method_calls_in_order do |expected|
attr_reader :actual
match do |actual|
@actual_err = actual
regexps = expected.map { |method_name| { name: method_name, regexp: make_regexp(method_name) } }
@actual = actual.backtrace.filter_map do |line|
regexp = regexps.find { |r| r[:regexp].match? line }
regexp[:name] if regexp
end
expected == @actual
end
diffable
failure_message do
"#{super()}\n\nFull error backtrace:\n #{@actual_err.backtrace.join("\n ")}"
end
private
def make_regexp(method_name)
Regexp.new("/spec/initializers/enumerator_next_patch_spec\\.rb:[0-9]+:in `#{method_name}'$")
end
end
def have_been_raised_by_next_and_not_fixed_up
contain_unique_method_calls_in_order %w(call_enum_method)
end
def have_been_raised_by_enum_object_and_fixed_up
contain_unique_method_calls_in_order %w(make_error call_enum_method)
end
def have_been_raised_by_nested_next_and_fixed_up
contain_unique_method_calls_in_order %w(call_nested_next call_enum_method)
end
methods = [
{
name: 'next',
expected_value: 'Test value'
},
{
name: 'next_values',
expected_value: ['Test value']
},
{
name: 'peek',
expected_value: 'Test value'
},
{
name: 'peek_values',
expected_value: ['Test value']
}
]
methods.each do |method|
describe "##{method[:name]}" do
def call_enum_method
enumerator.send(method_name)
end
let(:method_name) { method[:name] }
subject { call_enum_method }
describe 'normal yield' do
let(:enumerator) { Enumerator.new { |yielder| yielder << 'Test value' } }
it 'returns yielded value' do
is_expected.to eq(method[:expected_value])
end
end
describe 'end of iteration' do
let(:enumerator) { Enumerator.new { |_| } }
it 'does not fix up StopIteration' do
expect { subject }.to raise_error do |err|
expect(err).to be_a(StopIteration)
expect(err).to have_been_raised_by_next_and_not_fixed_up
end
end
context 'nested enum object' do
def call_nested_next
nested_enumerator.next
end
let(:nested_enumerator) { Enumerator.new { |_| } }
let(:enumerator) { Enumerator.new { |yielder| yielder << call_nested_next } }
it 'fixes up StopIteration thrown by another instance of #next' do
expect { subject }.to raise_error do |err|
expect(err).to be_a(StopIteration)
expect(err).to have_been_raised_by_nested_next_and_fixed_up
end
end
end
end
describe 'arguments error' do
def call_enum_method
enumerator.send(method_name, 'extra argument')
end
let(:enumerator) { Enumerator.new { |_| } }
it 'does not fix up ArgumentError' do
expect { subject }.to raise_error do |err|
expect(err).to be_a(ArgumentError)
expect(err).to have_been_raised_by_next_and_not_fixed_up
end
end
end
describe 'error' do
let(:enumerator) { Enumerator.new { |_| raise error } }
let(:error) { make_error }
it 'fixes up StopIteration' do
def make_error
StopIteration.new.tap { |err| err.set_backtrace(caller) }
end
expect { subject }.to raise_error do |err|
expect(err).to be(error)
expect(err).to have_been_raised_by_enum_object_and_fixed_up
end
end
it 'fixes up ArgumentError' do
def make_error
ArgumentError.new.tap { |err| err.set_backtrace(caller) }
end
expect { subject }.to raise_error do |err|
expect(err).to be(error)
expect(err).to have_been_raised_by_enum_object_and_fixed_up
end
end
it 'adds backtrace from other errors' do
def make_error
StandardError.new('This is a test').tap { |err| err.set_backtrace(caller) }
end
expect { subject }.to raise_error do |err|
expect(err).to be(error)
expect(err).to have_been_raised_by_enum_object_and_fixed_up
expect(err.message).to eq('This is a test')
end
end
end
end
end
end
end