kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2021-11-05 21:10:23 +00:00
parent 25307dda30
commit 331eae9a3e
22 changed files with 848 additions and 327 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app/assets/javascripts/vue_shared/security_reports/store/modules/sast/actions.js
View file

@ -18,6 +18,7 @@ export const fetchDiff = ({ state, rootState, dispatch }) => {
return fetchDiffData(rootState, state.paths.diffEndpoint, REPORT_TYPE_SAST)
.then((data) => {
dispatch('receiveDiffSuccess', data);
return data;
})
.catch(() => {
dispatch('receiveDiffError');

1
app/assets/javascripts/vue_shared/security_reports/store/modules/secret_detection/actions.js
View file

@ -18,6 +18,7 @@ export const fetchDiff = ({ state, rootState, dispatch }) => {
return fetchDiffData(rootState, state.paths.diffEndpoint, REPORT_TYPE_SECRET_DETECTION)
.then((data) => {
dispatch('receiveDiffSuccess', data);
return data;
})
.catch(() => {
dispatch('receiveDiffError');

21
app/graphql/mutations/security/ci_configuration/configure_sast_iac.rb Normal file
View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
module Mutations
module Security
module CiConfiguration
class ConfigureSastIac < BaseSecurityAnalyzer
graphql_name 'ConfigureSastIac'
description <<~DESC
Enable SAST IaC for a project in a new or
modified `.gitlab-ci.yml` file in a new branch. The new
branch and a URL to create a merge request are a part of the
response.
DESC
def configure_analyzer(project, **_args)
::Security::CiConfiguration::SastIacCreateService.new(project, current_user).execute
end
end
end
end
end

1
app/graphql/types/mutation_type.rb
View file

@ -16,6 +16,7 @@ module Types
mount_mutation Mutations::AlertManagement::HttpIntegration::ResetToken
mount_mutation Mutations::AlertManagement::HttpIntegration::Destroy
mount_mutation Mutations::Security::CiConfiguration::ConfigureSast
mount_mutation Mutations::Security::CiConfiguration::ConfigureSastIac
mount_mutation Mutations::Security::CiConfiguration::ConfigureSecretDetection
mount_mutation Mutations::AlertManagement::PrometheusIntegration::Create
mount_mutation Mutations::AlertManagement::PrometheusIntegration::Update

25
app/services/security/ci_configuration/sast_iac_create_service.rb Normal file
View file

@ -0,0 +1,25 @@
# frozen_string_literal: true
module Security
module CiConfiguration
class SastIacCreateService < ::Security::CiConfiguration::BaseCreateService
private
def action
Security::CiConfiguration::SastIacBuildAction.new(project.auto_devops_enabled?, existing_gitlab_ci_content).generate
end
def next_branch
'set-sast-iac-config'
end
def message
_('Configure SAST IaC in `.gitlab-ci.yml`, creating this file if it does not already exist')
end
def description
_('Configure SAST IaC in `.gitlab-ci.yml` using the GitLab managed template. You can [add variable overrides](https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings) to customize SAST IaC settings.')
end
end
end
end

25
doc/api/graphql/reference/index.md
View file

@ -1009,6 +1009,31 @@ Input type: `ConfigureSastInput`
| <a id="mutationconfiguresasterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
| <a id="mutationconfiguresastsuccesspath"></a>`successPath` | [`String`](#string) | Redirect path to use when the response is successful. |
### `Mutation.configureSastIac`
Enable SAST IaC for a project in a new or
modified `.gitlab-ci.yml` file in a new branch. The new
branch and a URL to create a merge request are a part of the
response.
Input type: `ConfigureSastIacInput`
#### Arguments
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="mutationconfiguresastiacclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
| <a id="mutationconfiguresastiacprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project. |
#### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="mutationconfiguresastiacbranch"></a>`branch` | [`String`](#string) | Branch that has the new/modified `.gitlab-ci.yml` file. |
| <a id="mutationconfiguresastiacclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
| <a id="mutationconfiguresastiacerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
| <a id="mutationconfiguresastiacsuccesspath"></a>`successPath` | [`String`](#string) | Redirect path to use when the response is successful. |
### `Mutation.configureSecretDetection`
Configure Secret Detection for a project by enabling Secret Detection

4
doc/ci/runners/build_cloud/linux_build_cloud.md
View file

@ -4,9 +4,9 @@ group: Runner
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Build Cloud runners for Linux **(FREE)**
# Runner Cloud for Linux **(FREE)**
GitLab Build Cloud runners for Linux run in autoscale mode and are powered by Google Cloud Platform.
Runner Cloud runners for Linux run in autoscale mode and are powered by Google Cloud Platform.
Autoscaling means reduced queue times to spin up CI/CD jobs, and isolated VMs for each job, thus maximizing security. These shared runners are available on GitLab.com.

4
doc/ci/runners/build_cloud/macos_build_cloud.md
View file

@ -4,9 +4,9 @@ group: Runner
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Build Cloud runners for macOS (Beta) **(FREE SAAS)**
# Runner Cloud for macOS (Beta) **(FREE SAAS)**
The GitLab Build Cloud for macOS Beta provides on-demand runners integrated with GitLab SaaS [CI/CD](../../../ci/index.md).
The Runner Cloud for macOS Beta provides on-demand runners integrated with GitLab SaaS [CI/CD](../../../ci/index.md).
Use these runners to build, test, and deploy apps for the Apple ecosystem (macOS, iOS, tvOS). You can take advantage
of all the capabilities of the GitLab single DevOps platform and not have to manage or operate a
build environment.

4
doc/ci/runners/build_cloud/windows_build_cloud.md
View file

@ -4,9 +4,9 @@ group: Runner
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Build Cloud runners for Windows (beta) **(FREE)**
# Runner Cloud for Windows (beta) **(FREE)**
GitLab Build Cloud runners for Windows are in [beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#beta)
Runner Cloud runners for Windows are in [beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#beta)
and shouldn't be used for production workloads.
During this beta period, the [shared runner pipeline quota](../../../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota)

2
doc/ci/runners/index.md
View file

@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
type: reference
---
# GitLab Build Cloud runners **(FREE)**
# GitLab Runner Cloud **(FREE)**
If you are using self-managed GitLab or you want to use your own runners on GitLab.com, you can
[install and configure your own runners](https://docs.gitlab.com/runner/install/).

102
lib/gitlab/background_migration.rb
View file

@ -2,8 +2,12 @@
module Gitlab
module BackgroundMigration
def self.queue
@queue ||= BackgroundMigrationWorker.sidekiq_options['queue']
def self.coordinator_for_database(database)
JobCoordinator.for_database(database)
end
def self.queue(database: :main)
coordinator_for_database(database).queue
end
# Begins stealing jobs from the background migrations queue, blocking the
@ -16,35 +20,10 @@ module Gitlab
# re-raises the exception.
#
# steal_class - The name of the class for which to steal jobs.
def self.steal(steal_class, retry_dead_jobs: false)
queues = [
Sidekiq::ScheduledSet.new,
Sidekiq::Queue.new(self.queue)
]
if retry_dead_jobs
queues << Sidekiq::RetrySet.new
queues << Sidekiq::DeadSet.new
end
queues.each do |queue|
queue.each do |job|
migration_class, migration_args = job.args
next unless job.klass == 'BackgroundMigrationWorker'
next unless migration_class == steal_class
next if block_given? && !(yield job)
begin
perform(migration_class, migration_args) if job.delete
rescue Exception # rubocop:disable Lint/RescueException
BackgroundMigrationWorker # enqueue this migration again
.perform_async(migration_class, migration_args)
raise
end
end
end
# retry_dead_jobs - Flag to control whether jobs in Sidekiq::RetrySet or Sidekiq::DeadSet are retried.
# database - tracking database this migration executes against
def self.steal(steal_class, retry_dead_jobs: false, database: :main, &block)
coordinator_for_database(database).steal(steal_class, retry_dead_jobs: retry_dead_jobs, &block)
end
##
@ -55,64 +34,13 @@ module Gitlab
#
# arguments - The arguments to pass to the background migration's "perform"
# method.
def self.perform(class_name, arguments)
migration_class_for(class_name).new.perform(*arguments)
# database - tracking database this migration executes against
def self.perform(class_name, arguments, database: :main)
coordinator_for_database(database).perform(class_name, arguments)
end
def self.remaining
enqueued = Sidekiq::Queue.new(self.queue)
scheduled = Sidekiq::ScheduledSet.new
[enqueued, scheduled].sum do |set|
set.count do |job|
job.klass == 'BackgroundMigrationWorker'
end
end
end
def self.exists?(migration_class, additional_queues = [])
enqueued = Sidekiq::Queue.new(self.queue)
scheduled = Sidekiq::ScheduledSet.new
enqueued_job?([enqueued, scheduled], migration_class)
end
def self.dead_jobs?(migration_class)
dead_set = Sidekiq::DeadSet.new
enqueued_job?([dead_set], migration_class)
end
def self.retrying_jobs?(migration_class)
retry_set = Sidekiq::RetrySet.new
enqueued_job?([retry_set], migration_class)
end
def self.migration_class_for(class_name)
# We don't pass class name with Gitlab::BackgroundMigration:: prefix anymore
# but some jobs could be already spawned so we need to have some backward compatibility period.
# Can be removed since 13.x
full_class_name_prefix_regexp = /\A(::)?Gitlab::BackgroundMigration::/
if class_name.match(full_class_name_prefix_regexp)
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
StandardError.new("Full class name is used"),
class_name: class_name
)
class_name = class_name.sub(full_class_name_prefix_regexp, '')
end
const_get(class_name, false)
end
def self.enqueued_job?(queues, migration_class)
queues.any? do |queue|
queue.any? do |job|
job.klass == 'BackgroundMigrationWorker' && job.args.first == migration_class
end
end
def self.exists?(migration_class, additional_queues = [], database: :main)
coordinator_for_database(database).exists?(migration_class, additional_queues) # rubocop:disable CodeReuse/ActiveRecord
end
end
end

127
lib/gitlab/background_migration/job_coordinator.rb Normal file
View file

@ -0,0 +1,127 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class responsible for executing background migrations based on the given database.
#
# Chooses the correct worker class when selecting jobs from the queue based on the
# convention of how the queues and worker classes are setup for each database.
#
# Also provides a database connection to the correct tracking database.
class JobCoordinator
VALID_DATABASES = %i[main].freeze
WORKER_CLASS_NAME = 'BackgroundMigrationWorker'
def self.for_database(database)
database = database.to_sym
unless VALID_DATABASES.include?(database)
raise ArgumentError, "database must be one of [#{VALID_DATABASES.join(', ')}], got '#{database}'"
end
namespace = database.to_s.capitalize unless database == :main
namespaced_worker_class = [namespace, WORKER_CLASS_NAME].compact.join('::')
new(database, "::#{namespaced_worker_class}".constantize)
end
attr_reader :database, :worker_class
def queue
@queue ||= worker_class.sidekiq_options['queue']
end
def with_shared_connection(&block)
Gitlab::Database::SharedModel.using_connection(connection, &block)
end
def steal(steal_class, retry_dead_jobs: false)
queues = [
Sidekiq::ScheduledSet.new,
Sidekiq::Queue.new(self.queue)
]
if retry_dead_jobs
queues << Sidekiq::RetrySet.new
queues << Sidekiq::DeadSet.new
end
queues.each do |queue|
queue.each do |job|
migration_class, migration_args = job.args
next unless job.klass == worker_class.name
next unless migration_class == steal_class
next if block_given? && !(yield job)
begin
perform(migration_class, migration_args) if job.delete
rescue Exception # rubocop:disable Lint/RescueException
worker_class # enqueue this migration again
.perform_async(migration_class, migration_args)
raise
end
end
end
end
def perform(class_name, arguments)
migration_class_for(class_name).new.perform(*arguments)
end
def remaining
enqueued = Sidekiq::Queue.new(self.queue)
scheduled = Sidekiq::ScheduledSet.new
[enqueued, scheduled].sum do |set|
set.count do |job|
job.klass == worker_class.name
end
end
end
def exists?(migration_class, additional_queues = [])
enqueued = Sidekiq::Queue.new(self.queue)
scheduled = Sidekiq::ScheduledSet.new
enqueued_job?([enqueued, scheduled], migration_class)
end
def dead_jobs?(migration_class)
dead_set = Sidekiq::DeadSet.new
enqueued_job?([dead_set], migration_class)
end
def retrying_jobs?(migration_class)
retry_set = Sidekiq::RetrySet.new
enqueued_job?([retry_set], migration_class)
end
def migration_class_for(class_name)
Gitlab::BackgroundMigration.const_get(class_name, false)
end
def enqueued_job?(queues, migration_class)
queues.any? do |queue|
queue.any? do |job|
job.klass == worker_class.name && job.args.first == migration_class
end
end
end
private
def initialize(database, worker_class)
@database = database
@worker_class = worker_class
end
def connection
@connection ||= Gitlab::Database.databases.fetch(database, Gitlab::Database.main).scope.connection
end
end
end
end

19
lib/security/ci_configuration/sast_iac_build_action.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Security
module CiConfiguration
class SastIacBuildAction < BaseBuildAction
private
def update_existing_content!
@existing_gitlab_ci_content['include'] = generate_includes
end
def template
return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled
'Security/SAST-IaC.latest.gitlab-ci.yml'
end
end
end
end

11
locale/gitlab.pot
View file

@ -8671,6 +8671,12 @@ msgstr ""
msgid "Configure Prometheus"
msgstr ""
msgid "Configure SAST IaC in `.gitlab-ci.yml` using the GitLab managed template. You can [add variable overrides](https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings) to customize SAST IaC settings."
msgstr ""
msgid "Configure SAST IaC in `.gitlab-ci.yml`, creating this file if it does not already exist"
msgstr ""
msgid "Configure SAST in `.gitlab-ci.yml` using the GitLab managed template. You can [add variable overrides](https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings) to customize SAST settings."
msgstr ""
@ -13044,16 +13050,13 @@ msgstr ""
msgid "EnvironmentsDashboard|More actions"
msgstr ""
msgid "EnvironmentsDashboard|More information"
msgstr ""
msgid "EnvironmentsDashboard|Remove"
msgstr ""
msgid "EnvironmentsDashboard|The environments dashboard provides a summary of each project's environments' status, including pipeline and alert statuses."
msgstr ""
msgid "EnvironmentsDashboard|This dashboard displays 3 environments per project, and is linked to the Operations Dashboard. When you add or remove a project from one dashboard, GitLab adds or removes the project from the other. %{readMoreLink}"
msgid "EnvironmentsDashboard|This dashboard displays 3 environments per project, and is linked to the Operations Dashboard. When you add or remove a project from one dashboard, GitLab adds or removes the project from the other. %{linkStart}More information%{linkEnd}"
msgstr ""
msgid "Environments|An error occurred while canceling the auto stop, please try again"

13
spec/graphql/mutations/security/ci_configuration/configure_sast_iac_spec.rb Normal file
View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Mutations::Security::CiConfiguration::ConfigureSastIac do
include GraphqlHelpers
let(:service) { ::Security::CiConfiguration::SastIacCreateService }
subject { resolve(described_class, args: { project_path: project.full_path }, ctx: { current_user: user }) }
include_examples 'graphql mutations security ci configuration'
end

318
spec/lib/gitlab/background_migration/job_coordinator_spec.rb Normal file
View file

@ -0,0 +1,318 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::JobCoordinator do
let(:database) { :main }
let(:worker_class) { BackgroundMigrationWorker }
let(:coordinator) { described_class.new(database, worker_class) }
describe '.for_database' do
it 'returns an executor with the correct worker class and database' do
coordinator = described_class.for_database(database)
expect(coordinator.database).to eq(database)
expect(coordinator.worker_class).to eq(worker_class)
end
context 'when passed in as a string' do
it 'retruns an executor with the correct worker class and database' do
coordinator = described_class.for_database(database.to_s)
expect(coordinator.database).to eq(database)
expect(coordinator.worker_class).to eq(worker_class)
end
end
context 'when an invalid value is given' do
it 'raises an error' do
expect do
described_class.for_database('notvalid')
end.to raise_error(ArgumentError, "database must be one of [main], got 'notvalid'")
end
end
end
describe '#queue' do
it 'returns background migration worker queue' do
expect(coordinator.queue).to eq(worker_class.sidekiq_options['queue'])
end
end
describe '#with_shared_connection' do
it 'yields to the block after properly configuring SharedModel' do
expect(Gitlab::Database::SharedModel).to receive(:using_connection)
.with(Gitlab::Database.main.scope.connection).and_yield
expect { |b| coordinator.with_shared_connection(&b) }.to yield_with_no_args
end
end
describe '#steal' do
context 'when there are enqueued jobs present' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: worker_class.name),
double(args: ['Bar', [20, 30]], klass: worker_class.name),
double(args: ['Foo', [20, 30]], klass: 'MergeWorker')
]
end
before do
allow(Sidekiq::Queue).to receive(:new)
.with(coordinator.queue)
.and_return(queue)
end
context 'when queue contains unprocessed jobs' do
it 'steals jobs from a queue' do
expect(queue[0]).to receive(:delete).and_return(true)
expect(coordinator).to receive(:perform).with('Foo', [10, 20])
coordinator.steal('Foo')
end
it 'does not steal job that has already been taken' do
expect(queue[0]).to receive(:delete).and_return(false)
expect(coordinator).not_to receive(:perform)
coordinator.steal('Foo')
end
it 'does not steal jobs for a different migration' do
expect(coordinator).not_to receive(:perform)
expect(queue[0]).not_to receive(:delete)
coordinator.steal('Baz')
end
context 'when a custom predicate is given' do
it 'steals jobs that match the predicate' do
expect(queue[0]).to receive(:delete).and_return(true)
expect(coordinator).to receive(:perform).with('Foo', [10, 20])
coordinator.steal('Foo') { |job| job.args.second.first == 10 && job.args.second.second == 20 }
end
it 'does not steal jobs that do not match the predicate' do
expect(described_class).not_to receive(:perform)
expect(queue[0]).not_to receive(:delete)
coordinator.steal('Foo') { |(arg1, _)| arg1 == 5 }
end
end
end
context 'when one of the jobs raises an error' do
let(:migration) { spy(:migration) }
let(:queue) do
[double(args: ['Foo', [10, 20]], klass: worker_class.name),
double(args: ['Foo', [20, 30]], klass: worker_class.name)]
end
before do
stub_const('Gitlab::BackgroundMigration::Foo', migration)
allow(queue[0]).to receive(:delete).and_return(true)
allow(queue[1]).to receive(:delete).and_return(true)
end
it 'enqueues the migration again and re-raises the error' do
allow(migration).to receive(:perform).with(10, 20).and_raise(Exception, 'Migration error').once
expect(worker_class).to receive(:perform_async).with('Foo', [10, 20]).once
expect { coordinator.steal('Foo') }.to raise_error(Exception)
end
end
end
context 'when there are scheduled jobs present', :redis do
it 'steals all jobs from the scheduled sets' do
Sidekiq::Testing.disable! do
worker_class.perform_in(10.minutes, 'Object')
expect(Sidekiq::ScheduledSet.new).to be_one
expect(coordinator).to receive(:perform).with('Object', any_args)
coordinator.steal('Object')
expect(Sidekiq::ScheduledSet.new).to be_none
end
end
end
context 'when there are enqueued and scheduled jobs present', :redis do
it 'steals from the scheduled sets queue first' do
Sidekiq::Testing.disable! do
expect(coordinator).to receive(:perform).with('Object', [1]).ordered
expect(coordinator).to receive(:perform).with('Object', [2]).ordered
worker_class.perform_async('Object', [2])
worker_class.perform_in(10.minutes, 'Object', [1])
coordinator.steal('Object')
end
end
end
context 'when retry_dead_jobs is true', :redis do
let(:retry_queue) do
[double(args: ['Object', [3]], klass: worker_class.name, delete: true)]
end
let(:dead_queue) do
[double(args: ['Object', [4]], klass: worker_class.name, delete: true)]
end
before do
allow(Sidekiq::RetrySet).to receive(:new).and_return(retry_queue)
allow(Sidekiq::DeadSet).to receive(:new).and_return(dead_queue)
end
it 'steals from the dead and retry queue' do
Sidekiq::Testing.disable! do
expect(coordinator).to receive(:perform).with('Object', [1]).ordered
expect(coordinator).to receive(:perform).with('Object', [2]).ordered
expect(coordinator).to receive(:perform).with('Object', [3]).ordered
expect(coordinator).to receive(:perform).with('Object', [4]).ordered
worker_class.perform_async('Object', [2])
worker_class.perform_in(10.minutes, 'Object', [1])
coordinator.steal('Object', retry_dead_jobs: true)
end
end
end
end
describe '#perform' do
let(:migration) { spy(:migration) }
before do
stub_const('Gitlab::BackgroundMigration::Foo', migration)
end
it 'performs a background migration' do
expect(migration).to receive(:perform).with(10, 20).once
coordinator.perform('Foo', [10, 20])
end
end
describe '.remaining', :redis do
context 'when there are jobs remaining' do
before do
Sidekiq::Testing.disable! do
MergeWorker.perform_async('Foo')
MergeWorker.perform_in(10.minutes, 'Foo')
5.times do
worker_class.perform_async('Foo')
end
3.times do
worker_class.perform_in(10.minutes, 'Foo')
end
end
end
it 'returns the enqueued jobs plus the scheduled jobs' do
expect(coordinator.remaining).to eq(8)
end
end
context 'when there are no jobs remaining' do
it 'returns zero' do
expect(coordinator.remaining).to be_zero
end
end
end
describe '.exists?', :redis do
context 'when there are enqueued jobs present' do
before do
Sidekiq::Testing.disable! do
MergeWorker.perform_async('Bar')
worker_class.perform_async('Foo')
end
end
it 'returns true if specific job exists' do
expect(coordinator.exists?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(coordinator.exists?('Bar')).to eq(false)
end
end
context 'when there are scheduled jobs present' do
before do
Sidekiq::Testing.disable! do
MergeWorker.perform_in(10.minutes, 'Bar')
worker_class.perform_in(10.minutes, 'Foo')
end
end
it 'returns true if specific job exists' do
expect(coordinator.exists?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(coordinator.exists?('Bar')).to eq(false)
end
end
end
describe '.dead_jobs?' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: worker_class.name),
double(args: ['Bar'], klass: 'MergeWorker')
]
end
context 'when there are dead jobs present' do
before do
allow(Sidekiq::DeadSet).to receive(:new).and_return(queue)
end
it 'returns true if specific job exists' do
expect(coordinator.dead_jobs?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(coordinator.dead_jobs?('Bar')).to eq(false)
end
end
end
describe '.retrying_jobs?' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: worker_class.name),
double(args: ['Bar'], klass: 'MergeWorker')
]
end
context 'when there are dead jobs present' do
before do
allow(Sidekiq::RetrySet).to receive(:new).and_return(queue)
end
it 'returns true if specific job exists' do
expect(coordinator.retrying_jobs?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(coordinator.retrying_jobs?('Bar')).to eq(false)
end
end
end
end

245
spec/lib/gitlab/background_migration_spec.rb
View file

@ -3,6 +3,14 @@
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration do
let(:coordinator) { described_class::JobCoordinator.for_database(:main) }
before do
allow(described_class).to receive(:coordinator_for_database)
.with(:main)
.and_return(coordinator)
end
describe '.queue' do
it 'returns background migration worker queue' do
expect(described_class.queue)
@ -11,7 +19,7 @@ RSpec.describe Gitlab::BackgroundMigration do
end
describe '.steal' do
context 'when there are enqueued jobs present' do
context 'when the queue contains unprocessed jobs' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'),
@ -22,110 +30,34 @@ RSpec.describe Gitlab::BackgroundMigration do
before do
allow(Sidekiq::Queue).to receive(:new)
.with(described_class.queue)
.with(coordinator.queue)
.and_return(queue)
end
context 'when queue contains unprocessed jobs' do
it 'steals jobs from a queue' do
it 'uses the job executor to steal jobs' do
expect(queue[0]).to receive(:delete).and_return(true)
expect(coordinator).to receive(:steal).with('Foo', retry_dead_jobs: false).and_call_original
expect(coordinator).to receive(:perform).with('Foo', [10, 20])
described_class.steal('Foo')
end
context 'when a custom predicate is given' do
it 'steals jobs that match the predicate' do
expect(queue[0]).to receive(:delete).and_return(true)
expect(described_class).to receive(:perform)
.with('Foo', [10, 20])
expect(coordinator).to receive(:perform).with('Foo', [10, 20])
described_class.steal('Foo')
described_class.steal('Foo') { |job| job.args.second.first == 10 && job.args.second.second == 20 }
end
it 'does not steal job that has already been taken' do
expect(queue[0]).to receive(:delete).and_return(false)
expect(described_class).not_to receive(:perform)
described_class.steal('Foo')
end
it 'does not steal jobs for a different migration' do
expect(described_class).not_to receive(:perform)
it 'does not steal jobs that do not match the predicate' do
expect(coordinator).not_to receive(:perform)
expect(queue[0]).not_to receive(:delete)
described_class.steal('Baz')
end
context 'when a custom predicate is given' do
it 'steals jobs that match the predicate' do
expect(queue[0]).to receive(:delete).and_return(true)
expect(described_class).to receive(:perform)
.with('Foo', [10, 20])
described_class.steal('Foo') { |job| job.args.second.first == 10 && job.args.second.second == 20 }
end
it 'does not steal jobs that do not match the predicate' do
expect(described_class).not_to receive(:perform)
expect(queue[0]).not_to receive(:delete)
described_class.steal('Foo') { |(arg1, _)| arg1 == 5 }
end
end
end
context 'when one of the jobs raises an error' do
let(:migration) { spy(:migration) }
let(:queue) do
[double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'),
double(args: ['Foo', [20, 30]], klass: 'BackgroundMigrationWorker')]
end
before do
stub_const("#{described_class}::Foo", migration)
allow(queue[0]).to receive(:delete).and_return(true)
allow(queue[1]).to receive(:delete).and_return(true)
end
it 'enqueues the migration again and re-raises the error' do
allow(migration).to receive(:perform).with(10, 20)
.and_raise(Exception, 'Migration error').once
expect(BackgroundMigrationWorker).to receive(:perform_async)
.with('Foo', [10, 20]).once
expect { described_class.steal('Foo') }.to raise_error(Exception)
end
end
end
context 'when there are scheduled jobs present', :redis do
it 'steals all jobs from the scheduled sets' do
Sidekiq::Testing.disable! do
BackgroundMigrationWorker.perform_in(10.minutes, 'Object')
expect(Sidekiq::ScheduledSet.new).to be_one
expect(described_class).to receive(:perform).with('Object', any_args)
described_class.steal('Object')
expect(Sidekiq::ScheduledSet.new).to be_none
end
end
end
context 'when there are enqueued and scheduled jobs present', :redis do
it 'steals from the scheduled sets queue first' do
Sidekiq::Testing.disable! do
expect(described_class).to receive(:perform)
.with('Object', [1]).ordered
expect(described_class).to receive(:perform)
.with('Object', [2]).ordered
BackgroundMigrationWorker.perform_async('Object', [2])
BackgroundMigrationWorker.perform_in(10.minutes, 'Object', [1])
described_class.steal('Object')
described_class.steal('Foo') { |(arg1, _)| arg1 == 5 }
end
end
end
@ -146,14 +78,10 @@ RSpec.describe Gitlab::BackgroundMigration do
it 'steals from the dead and retry queue' do
Sidekiq::Testing.disable! do
expect(described_class).to receive(:perform)
.with('Object', [1]).ordered
expect(described_class).to receive(:perform)
.with('Object', [2]).ordered
expect(described_class).to receive(:perform)
.with('Object', [3]).ordered
expect(described_class).to receive(:perform)
.with('Object', [4]).ordered
expect(coordinator).to receive(:perform).with('Object', [1]).ordered
expect(coordinator).to receive(:perform).with('Object', [2]).ordered
expect(coordinator).to receive(:perform).with('Object', [3]).ordered
expect(coordinator).to receive(:perform).with('Object', [4]).ordered
BackgroundMigrationWorker.perform_async('Object', [2])
BackgroundMigrationWorker.perform_in(10.minutes, 'Object', [1])
@ -171,50 +99,12 @@ RSpec.describe Gitlab::BackgroundMigration do
stub_const("#{described_class.name}::Foo", migration)
end
it 'performs a background migration' do
it 'uses the job executor to perform a background migration' do
expect(coordinator).to receive(:perform).with('Foo', [10, 20]).and_call_original
expect(migration).to receive(:perform).with(10, 20).once
described_class.perform('Foo', [10, 20])
end
context 'backward compatibility' do
it 'performs a background migration for fully-qualified job classes' do
expect(migration).to receive(:perform).with(10, 20).once
expect(Gitlab::ErrorTracking)
.to receive(:track_and_raise_for_dev_exception)
.with(instance_of(StandardError), hash_including(:class_name))
described_class.perform('Gitlab::BackgroundMigration::Foo', [10, 20])
end
end
end
describe '.remaining', :redis do
context 'when there are jobs remaining' do
before do
Sidekiq::Testing.disable! do
MergeWorker.perform_async('Foo')
MergeWorker.perform_in(10.minutes, 'Foo')
5.times do
BackgroundMigrationWorker.perform_async('Foo')
end
3.times do
BackgroundMigrationWorker.perform_in(10.minutes, 'Foo')
end
end
end
it 'returns the enqueued jobs plus the scheduled jobs' do
expect(described_class.remaining).to eq(8)
end
end
context 'when there are no jobs remaining' do
it 'returns zero' do
expect(described_class.remaining).to be_zero
end
end
end
describe '.exists?', :redis do
@ -226,76 +116,17 @@ RSpec.describe Gitlab::BackgroundMigration do
end
end
it 'returns true if specific job exists' do
it 'uses the job executor to find if a job exists' do
expect(coordinator).to receive(:exists?).with('Foo', []).and_call_original
expect(described_class.exists?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
it 'uses the job executor to find a job does not exist' do
expect(coordinator).to receive(:exists?).with('Bar', []).and_call_original
expect(described_class.exists?('Bar')).to eq(false)
end
end
context 'when there are scheduled jobs present' do
before do
Sidekiq::Testing.disable! do
MergeWorker.perform_in(10.minutes, 'Bar')
BackgroundMigrationWorker.perform_in(10.minutes, 'Foo')
end
end
it 'returns true if specific job exists' do
expect(described_class.exists?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(described_class.exists?('Bar')).to eq(false)
end
end
end
describe '.dead_jobs?' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'),
double(args: ['Bar'], klass: 'MergeWorker')
]
end
context 'when there are dead jobs present' do
before do
allow(Sidekiq::DeadSet).to receive(:new).and_return(queue)
end
it 'returns true if specific job exists' do
expect(described_class.dead_jobs?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(described_class.dead_jobs?('Bar')).to eq(false)
end
end
end
describe '.retrying_jobs?' do
let(:queue) do
[
double(args: ['Foo', [10, 20]], klass: 'BackgroundMigrationWorker'),
double(args: ['Bar'], klass: 'MergeWorker')
]
end
context 'when there are dead jobs present' do
before do
allow(Sidekiq::RetrySet).to receive(:new).and_return(queue)
end
it 'returns true if specific job exists' do
expect(described_class.retrying_jobs?('Foo')).to eq(true)
end
it 'returns false if specific job does not exist' do
expect(described_class.retrying_jobs?('Bar')).to eq(false)
end
end
end
end

23
spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb
View file

@ -583,12 +583,33 @@ RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do
end
describe '#finalized_background_migration' do
include_context 'background migration job class'
let(:job_coordinator) { Gitlab::BackgroundMigration::JobCoordinator.new(:main, BackgroundMigrationWorker) }
let!(:job_class_name) { 'TestJob' }
let!(:job_class) { Class.new }
let!(:job_perform_method) do
->(*arguments) do
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
# Value is 'TestJob' defined by :job_class_name in the let! above.
# Scoping prohibits us from directly referencing job_class_name.
RSpec.current_example.example_group_instance.job_class_name,
arguments
)
end
end
let!(:tracked_pending_job) { create(:background_migration_job, class_name: job_class_name, status: :pending, arguments: [1]) }
let!(:tracked_successful_job) { create(:background_migration_job, class_name: job_class_name, status: :succeeded, arguments: [2]) }
before do
job_class.define_method(:perform, job_perform_method)
allow(Gitlab::BackgroundMigration).to receive(:coordinator_for_database)
.with(:main).and_return(job_coordinator)
expect(job_coordinator).to receive(:migration_class_for)
.with(job_class_name).at_least(:once) { job_class }
Sidekiq::Testing.disable! do
BackgroundMigrationWorker.perform_async(job_class_name, [1, 2])
BackgroundMigrationWorker.perform_async(job_class_name, [3, 4])

163
spec/lib/security/ci_configuration/sast_iac_build_action_spec.rb Normal file
View file

@ -0,0 +1,163 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::CiConfiguration::SastIacBuildAction do
subject(:result) { described_class.new(auto_devops_enabled, gitlab_ci_content).generate }
let(:params) { {} }
context 'with existing .gitlab-ci.yml' do
let(:auto_devops_enabled) { false }
context 'sast iac has not been included' do
let(:expected_yml) do
<<-CI_YML.strip_heredoc
# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
stages:
- test
- security
variables:
RANDOM: make sure this persists
include:
- template: existing.yml
- template: Security/SAST-IaC.latest.gitlab-ci.yml
CI_YML
end
context 'template includes are an array' do
let(:gitlab_ci_content) do
{ "stages" => %w(test security),
"variables" => { "RANDOM" => "make sure this persists" },
"include" => [{ "template" => "existing.yml" }] }
end
it 'generates the correct YML' do
expect(result[:action]).to eq('update')
expect(result[:content]).to eq(expected_yml)
end
end
context 'template include is not an array' do
let(:gitlab_ci_content) do
{ "stages" => %w(test security),
"variables" => { "RANDOM" => "make sure this persists" },
"include" => { "template" => "existing.yml" } }
end
it 'generates the correct YML' do
expect(result[:action]).to eq('update')
expect(result[:content]).to eq(expected_yml)
end
end
end
context 'secret_detection has been included' do
let(:expected_yml) do
<<-CI_YML.strip_heredoc
# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
stages:
- test
variables:
RANDOM: make sure this persists
include:
- template: Security/SAST-IaC.latest.gitlab-ci.yml
CI_YML
end
context 'secret_detection template include are an array' do
let(:gitlab_ci_content) do
{ "stages" => %w(test),
"variables" => { "RANDOM" => "make sure this persists" },
"include" => [{ "template" => "Security/SAST-IaC.latest.gitlab-ci.yml" }] }
end
it 'generates the correct YML' do
expect(result[:action]).to eq('update')
expect(result[:content]).to eq(expected_yml)
end
end
context 'secret_detection template include is not an array' do
let(:gitlab_ci_content) do
{ "stages" => %w(test),
"variables" => { "RANDOM" => "make sure this persists" },
"include" => { "template" => "Security/SAST-IaC.latest.gitlab-ci.yml" } }
end
it 'generates the correct YML' do
expect(result[:action]).to eq('update')
expect(result[:content]).to eq(expected_yml)
end
end
end
end
context 'with no .gitlab-ci.yml' do
let(:gitlab_ci_content) { nil }
context 'autodevops disabled' do
let(:auto_devops_enabled) { false }
let(:expected_yml) do
<<-CI_YML.strip_heredoc
# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
include:
- template: Security/SAST-IaC.latest.gitlab-ci.yml
CI_YML
end
it 'generates the correct YML' do
expect(result[:action]).to eq('create')
expect(result[:content]).to eq(expected_yml)
end
end
context 'with autodevops enabled' do
let(:auto_devops_enabled) { true }
let(:expected_yml) do
<<-CI_YML.strip_heredoc
# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
include:
- template: Auto-DevOps.gitlab-ci.yml
CI_YML
end
before do
allow_next_instance_of(described_class) do |sast_iac_build_actions|
allow(sast_iac_build_actions).to receive(:auto_devops_stages).and_return(fast_auto_devops_stages)
end
end
it 'generates the correct YML' do
expect(result[:action]).to eq('create')
expect(result[:content]).to eq(expected_yml)
end
end
end
# stubbing this method allows this spec file to use fast_spec_helper
def fast_auto_devops_stages
auto_devops_template = YAML.safe_load( File.read('lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml') )
auto_devops_template['stages']
end
end

26
spec/requests/api/graphql/mutations/security/ci_configuration/configure_sast_iac_spec.rb Normal file
View file

@ -0,0 +1,26 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'ConfigureSastIac' do
include GraphqlHelpers
let_it_be(:project) { create(:project, :test_repo) }
let(:variables) { { project_path: project.full_path } }
let(:mutation) { graphql_mutation(:configure_sast_iac, variables) }
let(:mutation_response) { graphql_mutation_response(:configureSastIac) }
context 'when authorized' do
let_it_be(:user) { project.owner }
it 'creates a branch with sast iac configured' do
post_graphql_mutation(mutation, current_user: user)
expect(response).to have_gitlab_http_status(:success)
expect(mutation_response['errors']).to be_empty
expect(mutation_response['branch']).not_to be_empty
expect(mutation_response['successPath']).not_to be_empty
end
end
end

19
spec/services/security/ci_configuration/sast_iac_create_service_spec.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::CiConfiguration::SastIacCreateService, :snowplow do
subject(:result) { described_class.new(project, user).execute }
let(:branch_name) { 'set-sast-iac-config-1' }
let(:snowplow_event) do
{
category: 'Security::CiConfiguration::SastIacCreateService',
action: 'create',
label: ''
}
end
include_examples 'services security ci configuration create service', true
end

21
spec/support/shared_contexts/lib/gitlab/database/background_migration_job_shared_context.rb
View file

@ -1,21 +0,0 @@
# frozen_string_literal: true
RSpec.shared_context 'background migration job class' do
let!(:job_class_name) { 'TestJob' }
let!(:job_class) { Class.new }
let!(:job_perform_method) do
->(*arguments) do
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
# Value is 'TestJob' defined by :job_class_name in the let! above.
# Scoping prohibits us from directly referencing job_class_name.
RSpec.current_example.example_group_instance.job_class_name,
arguments
)
end
end
before do
job_class.define_method(:perform, job_perform_method)
expect(Gitlab::BackgroundMigration).to receive(:migration_class_for).with(job_class_name).at_least(:once) { job_class }
end
end