Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2020-11-17 21:09:19 +00:00
parent c663374b3d
commit 1b2e02ede9
43 changed files with 851 additions and 50 deletions

View file

@ -26,9 +26,6 @@ rules:
- _links
import/no-unresolved:
- error
- ignore:
# https://gitlab.com/gitlab-org/gitlab/issues/38226
- '^ee_component/'
# Disabled for now, to make the airbnb-base 12.1.0 -> 13.1.0 update smoother
no-else-return:
- error

View file

@ -42,11 +42,11 @@ Graphql/IDType:
Graphql/ResolverType:
Exclude:
- 'app/graphql/resolvers/base_resolver.rb'
- 'app/graphql/resolvers/ci/jobs_resolver.rb'
- 'app/graphql/resolvers/ci/pipeline_stages_resolver.rb'
- 'app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb'
- 'app/graphql/resolvers/merge_requests_resolver.rb'
- 'app/graphql/resolvers/users/group_count_resolver.rb'
- 'ee/app/graphql/resolvers/ci/jobs_resolver.rb'
- 'ee/app/graphql/resolvers/geo/merge_request_diff_registries_resolver.rb'
- 'ee/app/graphql/resolvers/geo/package_file_registries_resolver.rb'
- 'ee/app/graphql/resolvers/geo/terraform_state_version_registries_resolver.rb'

View file

@ -1 +1 @@
f869e2716122b17ce78508aacf981a098406d2d7
381df30b1b49b9fcfbc1e4107a106a70f1403c7d

View file

@ -14,6 +14,7 @@ export default function initGFMInput($els) {
milestones: enableGFM,
mergeRequests: enableGFM,
labels: enableGFM,
vulnerabilities: enableGFM,
});
});
}

View file

@ -6,6 +6,7 @@ import SidebarMediator from '~/sidebar/sidebar_mediator';
import { isUserBusy } from '~/set_status_modal/utils';
import glRegexp from './lib/utils/regexp';
import AjaxCache from './lib/utils/ajax_cache';
import axios from '~/lib/utils/axios_utils';
import { spriteIcon } from './lib/utils/common_utils';
import * as Emoji from '~/emoji';
@ -55,6 +56,7 @@ export const defaultAutocompleteConfig = {
milestones: true,
labels: true,
snippets: true,
vulnerabilities: true,
};
class GfmAutoComplete {
@ -62,6 +64,7 @@ class GfmAutoComplete {
this.dataSources = dataSources;
this.cachedData = {};
this.isLoadingData = {};
this.previousQuery = '';
}
setup(input, enableMap = defaultAutocompleteConfig) {
@ -561,7 +564,7 @@ class GfmAutoComplete {
}
getDefaultCallbacks() {
const fetchData = this.fetchData.bind(this);
const self = this;
return {
sorter(query, items, searchKey) {
@ -574,7 +577,14 @@ class GfmAutoComplete {
},
filter(query, data, searchKey) {
if (GfmAutoComplete.isLoading(data)) {
fetchData(this.$inputor, this.at);
self.fetchData(this.$inputor, this.at);
return data;
} else if (
GfmAutoComplete.isTypeWithBackendFiltering(this.at) &&
self.previousQuery !== query
) {
self.fetchData(this.$inputor, this.at, query);
self.previousQuery = query;
return data;
}
return $.fn.atwho.default.callbacks.filter(query, data, searchKey);
@ -622,13 +632,22 @@ class GfmAutoComplete {
};
}
fetchData($input, at) {
fetchData($input, at, search) {
if (this.isLoadingData[at]) return;
this.isLoadingData[at] = true;
const dataSource = this.dataSources[GfmAutoComplete.atTypeMap[at]];
if (this.cachedData[at]) {
if (GfmAutoComplete.isTypeWithBackendFiltering(at)) {
axios
.get(dataSource, { params: { search } })
.then(({ data }) => {
this.loadData($input, at, data);
})
.catch(() => {
this.isLoadingData[at] = false;
});
} else if (this.cachedData[at]) {
this.loadData($input, at, this.cachedData[at]);
} else if (GfmAutoComplete.atTypeMap[at] === 'emojis') {
this.loadEmojiData($input, at).catch(() => {});
@ -714,7 +733,9 @@ class GfmAutoComplete {
// https://github.com/ichord/At.js
const atSymbolsWithBar = Object.keys(controllers)
.join('|')
.replace(/[$]/, '\\$&');
.replace(/[$]/, '\\$&')
.replace(/([[\]:])/g, '\\$1');
const atSymbolsWithoutBar = Object.keys(controllers).join('');
const targetSubtext = subtext.split(GfmAutoComplete.regexSubtext).pop();
const resultantFlag = flag.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&');
@ -745,9 +766,14 @@ GfmAutoComplete.atTypeMap = {
'~': 'labels',
'%': 'milestones',
'/': 'commands',
'[vulnerability:': 'vulnerabilities',
$: 'snippets',
};
GfmAutoComplete.typesWithBackendFiltering = ['vulnerabilities'];
GfmAutoComplete.isTypeWithBackendFiltering = type =>
GfmAutoComplete.typesWithBackendFiltering.includes(GfmAutoComplete.atTypeMap[type]);
function findEmoji(name) {
return Emoji.searchEmoji(name, { match: 'contains', raw: true }).sort((a, b) => {
if (a.index !== b.index) {

View file

@ -16,5 +16,6 @@ export default (initGFM = true) => {
milestones: initGFM,
labels: initGFM,
snippets: initGFM,
vulnerabilities: initGFM,
});
};

View file

@ -177,6 +177,7 @@ export default {
milestones: this.enableAutocomplete && !this.glFeatures.tributeAutocomplete,
labels: this.enableAutocomplete && !this.glFeatures.tributeAutocomplete,
snippets: this.enableAutocomplete && !this.glFeatures.tributeAutocomplete,
vulnerabilities: this.enableAutocomplete,
},
true,
);

View file

@ -39,7 +39,7 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController
private
def autocomplete_service
@autocomplete_service ||= ::Projects::AutocompleteService.new(@project, current_user)
@autocomplete_service ||= ::Projects::AutocompleteService.new(@project, current_user, params)
end
def target

View file

@ -300,13 +300,13 @@ class SessionsController < Devise::SessionsController
def authentication_method
if user_params[:otp_attempt]
"two-factor"
AuthenticationEvent::TWO_FACTOR
elsif user_params[:device_response] && Feature.enabled?(:webauthn)
"two-factor-via-webauthn-device"
AuthenticationEvent::TWO_FACTOR_WEBAUTHN
elsif user_params[:device_response] && !Feature.enabled?(:webauthn)
"two-factor-via-u2f-device"
AuthenticationEvent::TWO_FACTOR_U2F
else
"standard"
AuthenticationEvent::STANDARD
end
end

View file

@ -0,0 +1,71 @@
# frozen_string_literal: true
# Security::JobsFinder
#
# Abstract class encapsulating common logic for finding jobs (builds) that are related to the Secure products
# SAST, DAST, Dependency Scanning, Container Scanning and License Management, Coverage Fuzzing
#
# Arguments:
# params:
# pipeline: required, only jobs for the specified pipeline will be found
# job_types: required, array of job types that should be returned, defaults to all job types
module Security
class JobsFinder
attr_reader :pipeline
def self.allowed_job_types
# Example return: [:sast, :dast, :dependency_scanning, :container_scanning, :license_management, :coverage_fuzzing]
raise NotImplementedError, 'allowed_job_types must be overwritten to return an array of job types'
end
def initialize(pipeline:, job_types: [])
if self.class == Security::JobsFinder
raise NotImplementedError, 'This is an abstract class, please instantiate its descendants'
end
if job_types.empty?
@job_types = self.class.allowed_job_types
elsif valid_job_types?(job_types)
@job_types = job_types
else
raise ArgumentError, "job_types must be from the following: #{self.class.allowed_job_types}"
end
@pipeline = pipeline
end
def execute
return [] if @job_types.empty?
if Feature.enabled?(:ci_build_metadata_config)
find_jobs
else
find_jobs_legacy
end
end
private
def find_jobs
@pipeline.builds.with_secure_reports_from_config_options(@job_types)
end
def find_jobs_legacy
# the query doesn't guarantee accuracy, so we verify it here
legacy_jobs_query.select do |job|
@job_types.find { |job_type| job.options.dig(:artifacts, :reports, job_type) }
end
end
def legacy_jobs_query
@job_types.map do |job_type|
@pipeline.builds.with_secure_reports_from_options(job_type)
end.reduce(&:or)
end
def valid_job_types?(job_types)
(job_types - self.class.allowed_job_types).empty?
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
# Security::LicenseScanningJobsFinder
#
# Used to find jobs (builds) that are related to the License Management.
#
# Arguments:
# params:
# pipeline: required, only jobs for the specified pipeline will be found
# job_types: required, array of job types that should be returned, defaults to all job types
module Security
class LicenseComplianceJobsFinder < JobsFinder
def self.allowed_job_types
[:license_management, :license_scanning]
end
end
end

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
# Security::SecurityJobsFinder
#
# Used to find jobs (builds) that are related to the Secure products:
# SAST, DAST, Dependency Scanning and Container Scanning
#
# Arguments:
# params:
# pipeline: required, only jobs for the specified pipeline will be found
# job_types: required, array of job types that should be returned, defaults to all job types
module Security
class SecurityJobsFinder < JobsFinder
def self.allowed_job_types
[:sast, :dast, :dependency_scanning, :container_scanning, :secret_detection, :coverage_fuzzing, :api_fuzzing]
end
end
end

View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
module Resolvers
module Ci
class JobsResolver < BaseResolver
alias_method :pipeline, :object
argument :security_report_types, [Types::Security::ReportTypeEnum],
required: false,
description: 'Filter jobs by the type of security report they produce'
def resolve(security_report_types: [])
if security_report_types.present?
::Security::SecurityJobsFinder.new(
pipeline: pipeline,
job_types: security_report_types
).execute
else
pipeline.statuses
end
end
end
end
end

View file

@ -13,65 +13,87 @@ module Types
field :id, GraphQL::ID_TYPE, null: false,
description: 'ID of the pipeline'
field :iid, GraphQL::STRING_TYPE, null: false,
description: 'Internal ID of the pipeline'
field :sha, GraphQL::STRING_TYPE, null: false,
description: "SHA of the pipeline's commit"
field :before_sha, GraphQL::STRING_TYPE, null: true,
description: 'Base SHA of the source branch'
field :status, PipelineStatusEnum, null: false,
description: "Status of the pipeline (#{::Ci::Pipeline.all_state_names.compact.join(', ').upcase})"
field :detailed_status, Types::Ci::DetailedStatusType, null: false,
description: 'Detailed status of the pipeline',
resolve: -> (obj, _args, ctx) { obj.detailed_status(ctx[:current_user]) }
field :config_source, PipelineConfigSourceEnum, null: true,
description: "Config source of the pipeline (#{::Enums::Ci::Pipeline.config_sources.keys.join(', ').upcase})"
field :duration, GraphQL::INT_TYPE, null: true,
description: 'Duration of the pipeline in seconds'
field :coverage, GraphQL::FLOAT_TYPE, null: true,
description: 'Coverage percentage'
field :created_at, Types::TimeType, null: false,
description: "Timestamp of the pipeline's creation"
field :updated_at, Types::TimeType, null: false,
description: "Timestamp of the pipeline's last activity"
field :started_at, Types::TimeType, null: true,
description: 'Timestamp when the pipeline was started'
field :finished_at, Types::TimeType, null: true,
description: "Timestamp of the pipeline's completion"
field :committed_at, Types::TimeType, null: true,
description: "Timestamp of the pipeline's commit"
field :stages, Types::Ci::StageType.connection_type, null: true,
description: 'Stages of the pipeline',
extras: [:lookahead],
resolver: Resolvers::Ci::PipelineStagesResolver
field :user, Types::UserType, null: true,
description: 'Pipeline user',
resolve: -> (pipeline, _args, _context) { Gitlab::Graphql::Loaders::BatchModelLoader.new(User, pipeline.user_id).find }
field :retryable, GraphQL::BOOLEAN_TYPE,
description: 'Specifies if a pipeline can be retried',
method: :retryable?,
null: false
field :cancelable, GraphQL::BOOLEAN_TYPE,
description: 'Specifies if a pipeline can be canceled',
method: :cancelable?,
null: false
field :jobs,
::Types::Ci::JobType.connection_type,
null: true,
description: 'Jobs belonging to the pipeline',
method: :statuses
resolver: ::Resolvers::Ci::JobsResolver
field :source_job, Types::Ci::JobType, null: true,
description: 'Job where pipeline was triggered from'
field :downstream, Types::Ci::PipelineType.connection_type, null: true,
description: 'Pipelines this pipeline will trigger',
method: :triggered_pipelines_with_preloads
field :upstream, Types::Ci::PipelineType, null: true,
description: 'Pipeline that triggered the pipeline',
method: :triggered_by_pipeline
field :path, GraphQL::STRING_TYPE, null: true,
description: "Relative path to the pipeline's page",
resolve: -> (obj, _args, _ctx) { ::Gitlab::Routing.url_helpers.project_pipeline_path(obj.project, obj) }
field :project, Types::ProjectType, null: true,
description: 'Project the pipeline belongs to'
end

View file

@ -0,0 +1,15 @@
# frozen_string_literal: true
module Types
module Security
class ReportTypeEnum < BaseEnum
graphql_name 'SecurityReportTypeEnum'
::Security::SecurityJobsFinder.allowed_job_types.each do |report_type|
value report_type.upcase,
value: report_type,
description: "#{report_type.upcase.to_s.tr('_', ' ')} scan report"
end
end
end
end

View file

@ -159,6 +159,7 @@ module NotesHelper
members: autocomplete,
issues: autocomplete,
mergeRequests: autocomplete,
vulnerabilities: autocomplete,
epics: autocomplete,
milestones: autocomplete,
labels: autocomplete

View file

@ -3,6 +3,12 @@
class AuthenticationEvent < ApplicationRecord
include UsageStatistics
TWO_FACTOR = 'two-factor'.freeze
TWO_FACTOR_U2F = 'two-factor-via-u2f-device'.freeze
TWO_FACTOR_WEBAUTHN = 'two-factor-via-webauthn-device'.freeze
STANDARD = 'standard'.freeze
STATIC_PROVIDERS = [TWO_FACTOR, TWO_FACTOR_U2F, TWO_FACTOR_WEBAUTHN, STANDARD].freeze
belongs_to :user, optional: true
validates :provider, :user_name, :result, presence: true
@ -17,6 +23,6 @@ class AuthenticationEvent < ApplicationRecord
scope :ldap, -> { where('provider LIKE ?', 'ldap%')}
def self.providers
distinct.pluck(:provider)
STATIC_PROVIDERS | Devise.omniauth_providers.map(&:to_s)
end
end

View file

@ -22,7 +22,7 @@ module Mentionable
def self.default_pattern
strong_memoize(:default_pattern) do
issue_pattern = Issue.reference_pattern
link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic].map(&:link_reference_pattern).compact)
link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic, Vulnerability].map(&:link_reference_pattern).compact)
reference_pattern(link_patterns, issue_pattern)
end
end

View file

@ -2,6 +2,27 @@
# Placeholder class for model that is implemented in EE
class Vulnerability < ApplicationRecord
include IgnorableColumns
def self.link_reference_pattern
nil
end
def self.reference_prefix
'[vulnerability:'
end
def self.reference_prefix_escaped
'[vulnerability&lbrack;'
end
def self.reference_postfix
']'
end
def self.reference_postfix_escaped
'&rbrack;'
end
end
Vulnerability.prepend_if_ee('EE::Vulnerability')

View file

@ -10,11 +10,11 @@ module Packages
composer_json
::Packages::Package.transaction do
::Packages::Composer::Metadatum.upsert(
::Packages::Composer::Metadatum.upsert({
package_id: created_package.id,
target_sha: target,
composer_json: composer_json
)
})
end
end

View file

@ -0,0 +1,5 @@
---
title: Replace poorly performing auth event providers query in usage ping
merge_request: 47710
author:
type: fixed

View file

@ -0,0 +1,5 @@
---
title: Filter jobs by security report type in GraphQL
merge_request: 47095
author:
type: added

View file

@ -199,6 +199,7 @@ module Gitlab
config.assets.precompile << "page_bundles/terminal.css"
config.assets.precompile << "page_bundles/todos.css"
config.assets.precompile << "page_bundles/reports.css"
config.assets.precompile << "page_bundles/roadmap.css"
config.assets.precompile << "page_bundles/wiki.css"
config.assets.precompile << "page_bundles/xterm.css"
config.assets.precompile << "page_bundles/alert_management_settings.css"

View file

@ -829,11 +829,11 @@ If you changed the location of the Container Registry `config.yml`:
sudo gitlab-ctl registry-garbage-collect /path/to/config.yml
```
You may also [remove all unreferenced manifests](#removing-unused-layers-not-referenced-by-manifests),
You may also [remove all untagged manifests and unreferenced layers](#removing-untagged-manifests-and-unreferenced-layers),
although this is a way more destructive operation, and you should first
understand the implications.
### Removing unused layers not referenced by manifests
### Removing untagged manifests and unreferenced layers
> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/3097) in Omnibus GitLab 11.10.
@ -841,11 +841,11 @@ DANGER: **Warning:**
This is a destructive operation.
The GitLab Container Registry follows the same default workflow as Docker Distribution:
retain all layers, even ones that are unreferenced directly to allow all content
to be accessed using context addressable identifiers.
retain untagged manifests and all layers, even ones that are not referenced directly. All content
can be accessed by using context addressable identifiers.
However, in most workflows, you don't care about old layers if they are not directly
referenced by the registry tag. The `registry-garbage-collect` command supports the
However, in most workflows, you don't care about untagged manifests and old layers if they are not directly
referenced by a tagged manifest. The `registry-garbage-collect` command supports the
`-m` switch to allow you to remove all unreferenced manifests and layers that are
not directly accessible via `tag`:
@ -853,6 +853,8 @@ not directly accessible via `tag`:
sudo gitlab-ctl registry-garbage-collect -m
```
Without the `-m` flag, the Container Registry only removes layers that are not referenced by any manifest, tagged or not.
Since this is a way more destructive operation, this behavior is disabled by default.
You are likely expecting this way of operation, but before doing that, ensure
that you have backed up all registry data.
@ -946,6 +948,8 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
5 4 * * 0 root gitlab-ctl registry-garbage-collect
```
You may want to add the `-m` flag to [remove untagged manifests and unreferenced layers](#removing-untagged-manifests-and-unreferenced-layers).
## Troubleshooting
Before diving in to the following sections, here's some basic troubleshooting:

View file

@ -111,6 +111,7 @@ as shown in the following table:
| [Interaction with Vulnerabilities](#interacting-with-the-vulnerabilities) | **{dotted-circle}** | **{check-circle}** |
| [Access to Security Dashboard](#security-dashboard) | **{dotted-circle}** | **{check-circle}** |
| [Configure SAST in the UI](#configure-sast-in-the-ui) | **{dotted-circle}** | **{check-circle}** |
| [Customize SAST Rulesets](#customize-rulesets) | **{dotted-circle}** | **{check-circle}** |
## Contribute your scanner
@ -205,15 +206,21 @@ spotbugs-sast:
FAIL_NEVER: 1
```
### Custom rulesets **(ULTIMATE)**
### Customize rulesets **(ULTIMATE)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235382) in GitLab 13.5.
You can customize the default scanning rules provided with SAST's NodeJS-Scan and Gosec analyzers.
Customization allows you to exclude rules and modify the behavior of existing rules.
You can customize the default scanning rules provided by our SAST analyzers.
Ruleset customization supports two capabilities:
1. Disabling predefined rules
1. Modifying the default behavior of a given analyzer
These capabilities can be used simultaneously.
To customize the default scanning rules, create a file containing custom rules. These rules
are passed through to the analyzer's underlying scanner tool.
are passed through to the analyzer's underlying scanner tools.
To create a custom ruleset:
@ -221,6 +228,25 @@ To create a custom ruleset:
1. Create a custom ruleset file named `sast-ruleset.toml` in the `.gitlab` directory.
1. In the `sast-ruleset.toml` file, do one of the following:
- Disable predefined rules belonging to SAST analyzers. In this example, the disabled rules
belong to `eslint` and `sobelow` and have the corresponding identifiers `type` and `value`:
```toml
[eslint]
[[eslint.ruleset]]
disable = true
[eslint.ruleset.identifier]
type = "eslint_rule_id"
value = "security/detect-object-injection"
[sobelow]
[[sobelow.ruleset]]
disable = true
[sobelow.ruleset.identifier]
type = "sobelow_rule_id"
value = "sql_injection"
```
- Define a custom analyzer configuration. In this example, customized rules are defined for the
`nodejs-scan` scanner:

View file

@ -425,6 +425,7 @@ GFM recognizes the following:
| merge request | `!123` | `namespace/project!123` | `project!123` |
| snippet | `$123` | `namespace/project$123` | `project$123` |
| epic **(ULTIMATE)** | `&123` | `group1/subgroup&123` | |
| vulnerability **(ULTIMATE)** *(1)* | `[vulnerability:123]` | `[vulnerability:namespace/project/123]` | `[vulnerability:project/123]` |
| label by ID | `~123` | `namespace/project~123` | `project~123` |
| one-word label by name | `~bug` | `namespace/project~bug` | `project~bug` |
| multi-word label by name | `~"feature request"` | `namespace/project~"feature request"` | `project~"feature request"` |
@ -438,6 +439,26 @@ GFM recognizes the following:
| repository file line references | `[README](doc/README#L13)` | | |
| [alert](../operations/incident_management/alerts.md) | `^alert#123` | `namespace/project^alert#123` | `project^alert#123` |
1. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/281035) in GitLab 13.6.
The Vulnerability special references feature is under development but ready for production use.
It is deployed behind a feature flag that is **disabled by default**.
[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md)
can opt to enable it for your instance.
It's disabled on GitLab.com.
To disable it:
```ruby
Feature.disable(:vulnerability_special_references)
```
To enable it:
```ruby
Feature.enable(:vulnerability_special_references)
```
For example, referencing an issue by using `#123` will format the output as a link
to issue number 123 with text `#123`. Likewise, a link to issue number 123 will be
recognized and formatted with text `#123`.

View file

@ -447,7 +447,7 @@ For the project where it's defined, tags matching the regex pattern are removed.
The underlying layers and images remain.
To delete the underlying layers and images that aren't associated with any tags, administrators can use
[garbage collection](../../../administration/packages/container_registry.md#removing-unused-layers-not-referenced-by-manifests) with the `-m` switch.
[garbage collection](../../../administration/packages/container_registry.md#removing-untagged-manifests-and-unreferenced-layers) with the `-m` switch.
### Enable the cleanup policy

View file

@ -119,7 +119,7 @@ module Banzai
# Yields the link's URL and inner HTML whenever the node is a valid <a> tag.
def yield_valid_link(node)
link = CGI.unescape(node.attr('href').to_s)
link = unescape_link(node.attr('href').to_s)
inner_html = node.inner_html
return unless link.force_encoding('UTF-8').valid_encoding?
@ -127,6 +127,10 @@ module Banzai
yield link, inner_html
end
def unescape_link(href)
CGI.unescape(href)
end
def replace_text_when_pattern_matches(node, index, pattern)
return unless node.text =~ pattern

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
module Banzai
module Filter
# The actual filter is implemented in the EE mixin
class VulnerabilityReferenceFilter < IssuableReferenceFilter
self.reference_type = :vulnerability
def self.object_class
Vulnerability
end
private
def project
context[:project]
end
end
end
end
Banzai::Filter::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::VulnerabilityReferenceFilter')

View file

@ -0,0 +1,16 @@
# frozen_string_literal: true
module Banzai
module ReferenceParser
# The actual parser is implemented in the EE mixin
class VulnerabilityParser < IssuableParser
self.reference_type = :vulnerability
def records_for_nodes(_nodes)
{}
end
end
end
end
Banzai::ReferenceParser::VulnerabilityParser.prepend_if_ee('::EE::Banzai::ReferenceParser::VulnerabilityParser')

View file

@ -4,7 +4,7 @@ module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor < Banzai::ReferenceExtractor
REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project
merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze
merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze
attr_accessor :project, :current_user, :author
# This counter is increased by a number of references filtered out by
# banzai reference exctractor. Note that this counter is stateful and
@ -38,7 +38,7 @@ module Gitlab
end
REFERABLES.each do |type|
define_method("#{type}s") do
define_method(type.to_s.pluralize) do
@references[type] ||= references(type)
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::JobsFinder do
it 'is an abstract class that does not permit instantiation' do
expect { described_class.new(pipeline: nil) }.to raise_error(
NotImplementedError,
'This is an abstract class, please instantiate its descendants'
)
end
describe '.allowed_job_types' do
it 'must be implemented by child classes' do
expect { described_class.allowed_job_types }.to raise_error(
NotImplementedError,
'allowed_job_types must be overwritten to return an array of job types'
)
end
end
end

View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::LicenseComplianceJobsFinder do
it_behaves_like ::Security::JobsFinder, described_class.allowed_job_types
describe "#execute" do
subject { finder.execute }
let(:pipeline) { create(:ci_pipeline) }
let(:finder) { described_class.new(pipeline: pipeline) }
let!(:sast_build) { create(:ci_build, :sast, pipeline: pipeline) }
let!(:container_scanning_build) { create(:ci_build, :container_scanning, pipeline: pipeline) }
let!(:dast_build) { create(:ci_build, :dast, pipeline: pipeline) }
let!(:license_scanning_build) { create(:ci_build, :license_scanning, pipeline: pipeline) }
let!(:license_management_build) { create(:ci_build, :license_management, pipeline: pipeline) }
it 'returns only the license_scanning jobs' do
is_expected.to contain_exactly(license_scanning_build, license_management_build)
end
end
end

View file

@ -0,0 +1,47 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::SecurityJobsFinder do
it_behaves_like ::Security::JobsFinder, described_class.allowed_job_types
describe "#execute" do
let(:pipeline) { create(:ci_pipeline) }
let(:finder) { described_class.new(pipeline: pipeline) }
subject { finder.execute }
context 'with specific secure job types' do
let!(:sast_build) { create(:ci_build, :sast, pipeline: pipeline) }
let!(:container_scanning_build) { create(:ci_build, :container_scanning, pipeline: pipeline) }
let!(:dast_build) { create(:ci_build, :dast, pipeline: pipeline) }
let!(:secret_detection_build) { create(:ci_build, :secret_detection, pipeline: pipeline) }
let(:finder) { described_class.new(pipeline: pipeline, job_types: [:sast, :container_scanning, :secret_detection]) }
it 'returns only those requested' do
is_expected.to include(sast_build)
is_expected.to include(container_scanning_build)
is_expected.to include(secret_detection_build)
is_expected.not_to include(dast_build)
end
end
context 'with combination of security jobs and license management jobs' do
let!(:sast_build) { create(:ci_build, :sast, pipeline: pipeline) }
let!(:container_scanning_build) { create(:ci_build, :container_scanning, pipeline: pipeline) }
let!(:dast_build) { create(:ci_build, :dast, pipeline: pipeline) }
let!(:secret_detection_build) { create(:ci_build, :secret_detection, pipeline: pipeline) }
let!(:license_management_build) { create(:ci_build, :license_management, pipeline: pipeline) }
it 'returns only the security jobs' do
is_expected.to include(sast_build)
is_expected.to include(container_scanning_build)
is_expected.to include(dast_build)
is_expected.to include(secret_detection_build)
is_expected.not_to include(license_management_build)
end
end
end
end

View file

@ -8,15 +8,226 @@ import GfmAutoComplete, { membersBeforeSave } from 'ee_else_ce/gfm_auto_complete
import { TEST_HOST } from 'helpers/test_constants';
import { getJSONFixture } from 'helpers/fixtures';
import waitForPromises from 'jest/helpers/wait_for_promises';
import MockAdapter from 'axios-mock-adapter';
import AjaxCache from '~/lib/utils/ajax_cache';
import axios from '~/lib/utils/axios_utils';
const labelsFixture = getJSONFixture('autocomplete_sources/labels.json');
describe('GfmAutoComplete', () => {
const gfmAutoCompleteCallbacks = GfmAutoComplete.prototype.getDefaultCallbacks.call({
fetchData: () => {},
});
const fetchDataMock = { fetchData: jest.fn() };
let gfmAutoCompleteCallbacks = GfmAutoComplete.prototype.getDefaultCallbacks.call(fetchDataMock);
let atwhoInstance;
let sorterValue;
let filterValue;
describe('DefaultOptions.filter', () => {
let items;
beforeEach(() => {
jest.spyOn(fetchDataMock, 'fetchData');
jest.spyOn($.fn.atwho.default.callbacks, 'filter').mockImplementation(() => {});
});
describe('assets loading', () => {
beforeEach(() => {
atwhoInstance = { setting: {}, $inputor: 'inputor', at: '[vulnerability:' };
items = ['loading'];
filterValue = gfmAutoCompleteCallbacks.filter.call(atwhoInstance, '', items);
});
it('should call the fetchData function without query', () => {
expect(fetchDataMock.fetchData).toHaveBeenCalledWith('inputor', '[vulnerability:');
});
it('should not call the default atwho filter', () => {
expect($.fn.atwho.default.callbacks.filter).not.toHaveBeenCalled();
});
it('should return the passed unfiltered items', () => {
expect(filterValue).toEqual(items);
});
});
describe('backend filtering', () => {
beforeEach(() => {
atwhoInstance = { setting: {}, $inputor: 'inputor', at: '[vulnerability:' };
items = [];
});
describe('when previous query is different from current one', () => {
beforeEach(() => {
gfmAutoCompleteCallbacks = GfmAutoComplete.prototype.getDefaultCallbacks.call({
previousQuery: 'oldquery',
...fetchDataMock,
});
filterValue = gfmAutoCompleteCallbacks.filter.call(atwhoInstance, 'newquery', items);
});
it('should call the fetchData function with query', () => {
expect(fetchDataMock.fetchData).toHaveBeenCalledWith(
'inputor',
'[vulnerability:',
'newquery',
);
});
it('should not call the default atwho filter', () => {
expect($.fn.atwho.default.callbacks.filter).not.toHaveBeenCalled();
});
it('should return the passed unfiltered items', () => {
expect(filterValue).toEqual(items);
});
});
describe('when previous query is not different from current one', () => {
beforeEach(() => {
gfmAutoCompleteCallbacks = GfmAutoComplete.prototype.getDefaultCallbacks.call({
previousQuery: 'oldquery',
...fetchDataMock,
});
filterValue = gfmAutoCompleteCallbacks.filter.call(
atwhoInstance,
'oldquery',
items,
'searchKey',
);
});
it('should not call the fetchData function', () => {
expect(fetchDataMock.fetchData).not.toHaveBeenCalled();
});
it('should call the default atwho filter', () => {
expect($.fn.atwho.default.callbacks.filter).toHaveBeenCalledWith(
'oldquery',
items,
'searchKey',
);
});
});
});
});
describe('fetchData', () => {
const { fetchData } = GfmAutoComplete.prototype;
let mock;
beforeEach(() => {
mock = new MockAdapter(axios);
jest.spyOn(axios, 'get');
jest.spyOn(AjaxCache, 'retrieve');
});
afterEach(() => {
mock.restore();
});
describe('already loading data', () => {
beforeEach(() => {
const context = {
isLoadingData: { '[vulnerability:': true },
dataSources: {},
cachedData: {},
};
fetchData.call(context, {}, '[vulnerability:', '');
});
it('should not call either axios nor AjaxCache', () => {
expect(axios.get).not.toHaveBeenCalled();
expect(AjaxCache.retrieve).not.toHaveBeenCalled();
});
});
describe('backend filtering', () => {
describe('data is not in cache', () => {
let context;
beforeEach(() => {
context = {
isLoadingData: { '[vulnerability:': false },
dataSources: { vulnerabilities: 'vulnerabilities_autocomplete_url' },
cachedData: {},
};
});
it('should call axios with query', () => {
fetchData.call(context, {}, '[vulnerability:', 'query');
expect(axios.get).toHaveBeenCalledWith('vulnerabilities_autocomplete_url', {
params: { search: 'query' },
});
});
it.each([200, 500])('should set the loading state', async responseStatus => {
mock.onGet('vulnerabilities_autocomplete_url').replyOnce(responseStatus);
fetchData.call(context, {}, '[vulnerability:', 'query');
expect(context.isLoadingData['[vulnerability:']).toBe(true);
await waitForPromises();
expect(context.isLoadingData['[vulnerability:']).toBe(false);
});
});
describe('data is in cache', () => {
beforeEach(() => {
const context = {
isLoadingData: { '[vulnerability:': false },
dataSources: { vulnerabilities: 'vulnerabilities_autocomplete_url' },
cachedData: { '[vulnerability:': [{}] },
};
fetchData.call(context, {}, '[vulnerability:', 'query');
});
it('should anyway call axios with query ignoring cache', () => {
expect(axios.get).toHaveBeenCalledWith('vulnerabilities_autocomplete_url', {
params: { search: 'query' },
});
});
});
});
describe('frontend filtering', () => {
describe('data is not in cache', () => {
beforeEach(() => {
const context = {
isLoadingData: { '#': false },
dataSources: { issues: 'issues_autocomplete_url' },
cachedData: {},
};
fetchData.call(context, {}, '#', 'query');
});
it('should call AjaxCache', () => {
expect(AjaxCache.retrieve).toHaveBeenCalledWith('issues_autocomplete_url', true);
});
});
describe('data is in cache', () => {
beforeEach(() => {
const context = {
isLoadingData: { '#': false },
dataSources: { issues: 'issues_autocomplete_url' },
cachedData: { '#': [{}] },
loadData: () => {},
};
fetchData.call(context, {}, '#', 'query');
});
it('should not call AjaxCache', () => {
expect(AjaxCache.retrieve).not.toHaveBeenCalled();
});
});
});
});
describe('DefaultOptions.sorter', () => {
describe('assets loading', () => {
@ -154,7 +365,6 @@ describe('GfmAutoComplete', () => {
'я',
'.',
"'",
'+',
'-',
'_',
];

View file

@ -1,4 +1,4 @@
/* eslint-disable global-require */
/* eslint-disable global-require, import/no-unresolved */
import { memoize } from 'lodash';
export const getProject = () => require('test_fixtures/api/projects/get.json');

View file

@ -0,0 +1,40 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Resolvers::Ci::JobsResolver do
include GraphqlHelpers
let_it_be(:pipeline) { create(:ci_pipeline) }
before_all do
create(:ci_build, name: 'Normal job', pipeline: pipeline)
create(:ci_build, :sast, name: 'DAST job', pipeline: pipeline)
create(:ci_build, :dast, name: 'SAST job', pipeline: pipeline)
create(:ci_build, :container_scanning, name: 'Container scanning job', pipeline: pipeline)
end
describe '#resolve' do
context 'when security_report_types is empty' do
it "returns all of the pipeline's jobs" do
jobs = resolve(described_class, obj: pipeline, args: {}, ctx: {})
job_names = jobs.map(&:name)
expect(job_names).to contain_exactly('Normal job', 'DAST job', 'SAST job', 'Container scanning job')
end
end
context 'when security_report_types is present' do
it "returns the pipeline's jobs with the given security report types" do
report_types = [
::Types::Security::ReportTypeEnum.values['SAST'].value,
::Types::Security::ReportTypeEnum.values['DAST'].value
]
jobs = resolve(described_class, obj: pipeline, args: { security_report_types: report_types }, ctx: {})
job_names = jobs.map(&:name)
expect(job_names).to contain_exactly('DAST job', 'SAST job')
end
end
end
end

View file

@ -0,0 +1,11 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe GitlabSchema.types['SecurityReportTypeEnum'] do
it 'exposes all security report types' do
expect(described_class.values.keys).to contain_exactly(
*::Security::SecurityJobsFinder.allowed_job_types.map(&:to_s).map(&:upcase)
)
end
end

View file

@ -296,7 +296,7 @@ RSpec.describe Gitlab::ReferenceExtractor do
end
it 'returns all supported prefixes' do
expect(prefixes.keys.uniq).to match_array(%w(@ # ~ % ! $ & *iteration:))
expect(prefixes.keys.uniq).to match_array(%w(@ # ~ % ! $ & [vulnerability: *iteration:))
end
it 'does not allow one prefix for multiple referables if not allowed specificly' do

View file

@ -172,6 +172,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
omniauth:
{ providers: omniauth_providers }
)
allow(Devise).to receive(:omniauth_providers).and_return(%w(ldapmain ldapsecondary group_saml))
for_defined_days_back do
user = create(:user)
@ -190,14 +191,14 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
groups: 2,
users_created: 6,
omniauth_providers: ['google_oauth2'],
user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4 }
user_auth_by_provider: { 'group_saml' => 2, 'ldap' => 4, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 }
)
expect(described_class.usage_activity_by_stage_manage(described_class.last_28_days_time_period)).to include(
events: 1,
groups: 1,
users_created: 3,
omniauth_providers: ['google_oauth2'],
user_auth_by_provider: { 'group_saml' => 1, 'ldap' => 2 }
user_auth_by_provider: { 'group_saml' => 1, 'ldap' => 2, 'standard' => 0, 'two-factor' => 0, 'two-factor-via-u2f-device' => 0, "two-factor-via-webauthn-device" => 0 }
)
end

View file

@ -37,15 +37,11 @@ RSpec.describe AuthenticationEvent do
describe '.providers' do
before do
create(:authentication_event, provider: :ldapmain)
create(:authentication_event, provider: :google_oauth2)
create(:authentication_event, provider: :standard)
create(:authentication_event, provider: :standard)
create(:authentication_event, provider: :standard)
allow(Devise).to receive(:omniauth_providers).and_return(%w(ldapmain google_oauth2))
end
it 'returns an array of distinct providers' do
expect(described_class.providers).to match_array %w(ldapmain google_oauth2 standard)
expect(described_class.providers).to match_array %w(ldapmain google_oauth2 standard two-factor two-factor-via-u2f-device two-factor-via-webauthn-device)
end
end
end

View file

@ -56,6 +56,45 @@ RSpec.describe 'Query.project(fullPath).pipelines' do
end
end
describe '.jobs(securityReportTypes)' do
let_it_be(:query) do
%(
query {
project(fullPath: "#{project.full_path}") {
pipelines {
nodes {
jobs(securityReportTypes: [SAST]) {
nodes {
name
}
}
}
}
}
}
)
end
it 'fetches the jobs matching the report type filter' do
pipeline = create(:ci_pipeline, project: project)
create(:ci_build, :dast, name: 'DAST Job 1', pipeline: pipeline)
create(:ci_build, :sast, name: 'SAST Job 1', pipeline: pipeline)
post_graphql(query, current_user: first_user)
expect(response).to have_gitlab_http_status(:ok)
pipelines_data = graphql_data.dig('project', 'pipelines', 'nodes')
job_names = pipelines_data.map do |pipeline_data|
jobs_data = pipeline_data.dig('jobs', 'nodes')
jobs_data.map { |job_data| job_data['name'] }
end.flatten
expect(job_names).to contain_exactly('SAST Job 1')
end
end
describe 'upstream' do
let_it_be(:pipeline) { create(:ci_pipeline, project: project, user: first_user) }
let_it_be(:upstream_project) { create(:project, :repository, :public) }
@ -176,8 +215,6 @@ RSpec.describe 'Query.project(fullPath).pipelines' do
expect do
post_graphql(query, current_user: second_user)
end.not_to exceed_query_limit(control_count)
expect(response).to have_gitlab_http_status(:ok)
end
end
end

View file

@ -0,0 +1,87 @@
# frozen_string_literal: true
RSpec.shared_examples ::Security::JobsFinder do |default_job_types|
let(:pipeline) { create(:ci_pipeline) }
describe '#new' do
it "does not get initialized for unsupported job types" do
expect { described_class.new(pipeline: pipeline, job_types: [:abcd]) }.to raise_error(
ArgumentError,
"job_types must be from the following: #{default_job_types}"
)
end
end
describe '#execute' do
let(:finder) { described_class.new(pipeline: pipeline) }
subject { finder.execute }
shared_examples 'JobsFinder core functionality' do
context 'when the pipeline has no jobs' do
it { is_expected.to be_empty }
end
context 'when the pipeline has no Secure jobs' do
before do
create(:ci_build, pipeline: pipeline)
end
it { is_expected.to be_empty }
end
context 'when the pipeline only has jobs without report artifacts' do
before do
create(:ci_build, pipeline: pipeline, options: { artifacts: { file: 'test.file' } })
end
it { is_expected.to be_empty }
end
context 'when the pipeline only has jobs with reports unrelated to Secure products' do
before do
create(:ci_build, pipeline: pipeline, options: { artifacts: { reports: { file: 'test.file' } } })
end
it { is_expected.to be_empty }
end
context 'when the pipeline only has jobs with reports with paths similar but not identical to Secure reports' do
before do
create(:ci_build, pipeline: pipeline, options: { artifacts: { reports: { file: 'report:sast:result.file' } } })
end
it { is_expected.to be_empty }
end
context 'when there is more than one pipeline' do
let(:job_type) { default_job_types.first }
let!(:build) { create(:ci_build, job_type, pipeline: pipeline) }
before do
create(:ci_build, job_type, pipeline: create(:ci_pipeline))
end
it 'returns jobs associated with provided pipeline' do
is_expected.to eq([build])
end
end
end
context 'when using legacy CI build metadata config storage' do
before do
stub_feature_flags(ci_build_metadata_config: false)
end
it_behaves_like 'JobsFinder core functionality'
end
context 'when using the new CI build metadata config storage' do
before do
stub_feature_flags(ci_build_metadata_config: true)
end
it_behaves_like 'JobsFinder core functionality'
end
end
end