Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-06-03 00:09:04 +00:00
parent 6ddc820225
commit 94aee27731
26 changed files with 145 additions and 99 deletions

View file

@ -13,8 +13,8 @@ include:
.test_variables:
variables:
QA_DEBUG: "true"
QA_GENERATE_ALLURE_REPORT: "true"
COLORIZED_LOGS: "true"
GITLAB_USERNAME: "root"
GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
GITLAB_ADMIN_USERNAME: "root"

View file

@ -13,10 +13,11 @@ module Ci
def execute
return unless @user.present? && @user.can?(:update_runners_registration_token, scope)
if scope.respond_to?(:runners_registration_token)
case scope
when ::ApplicationSetting
scope.reset_runners_registration_token!
scope.runners_registration_token
else
ApplicationSetting.current_without_cache.runners_registration_token
when ::Group, ::Project
scope.reset_runners_token!
scope.runners_token
end

View file

@ -0,0 +1,37 @@
---
stage: Secure
group: Dynamic Analysis
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Mixed Content
## Description
The target application was found to request resources over insecure transport protocols (HTTP). This is usually due to HTML
elements which load resources using the `http://` scheme instead of `https://`. It should be noted that most modern browsers
block these requests automatically so there is limited risk.
Some parts of the application may not behave correctly since these files are not being properly loaded.
## Remediation
Ensure all HTML elements which load resources from a URL (JavaScript, stylesheets, images, video and other media) are set to
use the `https://` scheme instead of `http://`. Alternatively, developers may use the `//` scheme, which will only load resources
over the same protocol that the originating page was loaded.
A browser visiting the website `https://example.com` with the HTML loading a file using
`<script src="//example.com/cdn/bundle.js"></script>`, would ensure the `example.com/cdn/bundle.js` file was loaded over
HTTPS.
## Details
| ID | Aggregated | CWE | Type | Risk |
|:---|:--------|:--------|:--------|:--------|
| 319.1 | true | 319 | Passive | Info |
## Links
- [OWASP](https://owasp.org/www-community/vulnerabilities/Insecure_Transport)
- [CWE](https://cwe.mitre.org/data/definitions/319.html)
- [MDN](https://developer.mozilla.org/en-US/docs/Web/Security/Mixed_content)

View file

@ -19,6 +19,7 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne
| [16.6](16.6.md) | AspNetMvc header exposes version information | Low | Passive |
| [16.7](16.7.md) | Strict-Transport-Security header missing or invalid | Low | Passive |
| [200.1](200.1.md) | Exposure of sensitive information to an unauthorized actor (private IP address) | Low | Passive |
| [319.1](319.1.md) | Mixed Content | Info | Passive |
| [352.1](352.1.md) | Absence of anti-CSRF tokens | Medium | Passive |
| [359.1](359.1.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (credit card) | Medium | Passive |
| [359.2](359.2.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (United States social security number) | Medium | Passive |

View file

@ -197,7 +197,8 @@ Secret Detection can be customized by defining available CI/CD variables:
|-----------------------------------|---------------|-------------|
| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. |
| `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. |
| `SECRET_DETECTION_IMAGE_SUFFIX` | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. |
| `SECRET_DETECTION_IMAGE_SUFFIX` | "" | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. |
| `SECRET_DETECTION_LOG_OPTS` | "" | [`git log`](https://git-scm.com/docs/git-log) options used to define commit ranges. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350660) in GitLab 15.1.|
In previous GitLab versions, the following variables were also available:

View file

@ -27,7 +27,7 @@ In GitLab, you can:
For self-managed GitLab, before you can use GitLab for your Terraform state files:
- An administrator must [set up Terraform state storage](../../../administration/terraform_state.md).
- You must enable [Operations](../../../operations/index.md) for your project. Go to **Settings > General**,
- You must enable the **Infrastructure** menu for your project. Go to **Settings > General**,
expand **Visibility, project features, permissions**, and under **Operations**, turn on the toggle.
## Initialize a Terraform state as a backend by using GitLab CI/CD

1
qa/.gitignore vendored
View file

@ -1,4 +1,3 @@
tmp/
reports/
no_of_examples/

View file

@ -2,7 +2,7 @@
source 'https://rubygems.org'
gem 'gitlab-qa', require: 'gitlab/qa'
gem 'gitlab-qa', '~> 7', require: 'gitlab/qa'
gem 'activesupport', '~> 6.1.4.7' # This should stay in sync with the root's Gemfile
gem 'allure-rspec', '~> 2.16.0'
gem 'capybara', '~> 3.35.0'

View file

@ -121,7 +121,7 @@ GEM
gitlab (4.18.0)
httparty (~> 0.18)
terminal-table (>= 1.5.1)
gitlab-qa (7.24.4)
gitlab-qa (7.29.1)
activesupport (~> 6.1)
gitlab (~> 4.18.0)
http (~> 5.0)
@ -368,7 +368,7 @@ DEPENDENCIES
deprecation_toolkit (~> 1.5.1)
faker (~> 2.19, >= 2.19.0)
fog-google (~> 1.17)
gitlab-qa
gitlab-qa (~> 7)
influxdb-client (~> 1.17)
knapsack (~> 4.0)
octokit (~> 4.21)

View file

@ -5,10 +5,12 @@ require 'capybara/dsl'
module QA
module Page
class Base
prepend Support::Page::Logging if Runtime::Env.debug?
prepend Support::Page::Logging
include Capybara::DSL
include Scenario::Actable
include Support::WaitForRequests
extend Validatable
extend SingleForwardable

View file

@ -99,13 +99,13 @@ module QA
fabrication_time: fabrication_time
)
Runtime::Logger.debug do
Runtime::Logger.info do
msg = ["==#{'=' * parents.size}>"]
msg << "#{fabrication_http_method} a #{Rainbow(name).black.bg(:white)}"
msg << resource.identifier
msg << "as a dependency of #{parents.last}" if parents.any?
msg << "via #{fabrication_method}"
msg << "in #{fabrication_time} seconds"
msg << "in #{fabrication_time.round(2)} seconds"
msg.compact.join(' ')
end
@ -161,7 +161,7 @@ module QA
end
def visit!(skip_resp_code_check: false)
Runtime::Logger.debug("Visiting #{Rainbow(self.class.name).black.bg(:white)} at #{web_url}")
Runtime::Logger.info("Visiting #{Rainbow(self.class.name).black.bg(:white)} at #{web_url}")
# Just in case an async action is not yet complete
Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check)
@ -224,7 +224,7 @@ module QA
def remove_via_api!
super
Runtime::Logger.debug(["Removed a #{self.class.name}", identifier].compact.join(' '))
Runtime::Logger.info(["Removed a #{self.class.name}", identifier].compact.join(' '))
end
protected

View file

@ -9,7 +9,7 @@ module QA
module Members
def add_member(user, access_level = AccessLevel::DEVELOPER)
Support::Retrier.retry_until do
QA::Runtime::Logger.debug(%Q[Adding user #{user.username} to #{full_path} #{self.class.name}])
QA::Runtime::Logger.info(%(Adding user #{user.username} to #{full_path} #{self.class.name}))
response = post Runtime::API::Request.new(api_client, api_members_path).url, { user_id: user.id, access_level: access_level }
break true if response.code == QA::Support::API::HTTP_STATUS_CREATED
@ -18,7 +18,7 @@ module QA
end
def remove_member(user)
QA::Runtime::Logger.debug(%Q[Removing user #{user.username} from #{full_path} #{self.class.name}])
QA::Runtime::Logger.info(%(Removing user #{user.username} from #{full_path} #{self.class.name}))
delete Runtime::API::Request.new(api_client, "#{api_members_path}/#{user.id}").url
end
@ -29,7 +29,7 @@ module QA
def invite_group(group, access_level = AccessLevel::GUEST)
Support::Retrier.retry_until do
QA::Runtime::Logger.debug(%Q[Sharing #{self.class.name} with #{group.name}])
QA::Runtime::Logger.info(%(Sharing #{self.class.name} with #{group.name}))
response = post Runtime::API::Request.new(api_client, api_share_path).url, { group_id: group.id, group_access: access_level }
response.code == QA::Support::API::HTTP_STATUS_CREATED

View file

@ -90,21 +90,19 @@ module QA
"/runners/#{id}"
end
def api_get_path
end
def api_get_path; end
def api_post_path
"/runners"
end
def api_post_body
end
def api_post_body; end
private
def dump_logs
if @docker_container.running?
@docker_container.logs { |line| QA::Runtime::Logger.debug(line) }
@docker_container.logs
else
QA::Runtime::Logger.debug("No runner container found named #{name}")
end

View file

@ -73,10 +73,6 @@ module QA
ENV['QA_DEFAULT_BRANCH'] || 'main'
end
def log_destination
ENV['QA_LOG_PATH'] || $stdout
end
def colorized_logs?
enabled?(ENV['COLORIZED_LOGS'], default: false)
end

View file

@ -9,10 +9,14 @@ module QA
def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown
# Global logger instance
#
# @return [ActiveSupport::Logger]
def self.logger
@logger ||= Gitlab::QA::TestLogger.logger(
level: Runtime::Env.debug? ? ::Logger::DEBUG : ::Logger::INFO,
source: 'QA Tests'
level: Runtime::Env.debug? ? "DEBUG" : Gitlab::QA::Runtime::Env.log_level,
source: 'QA Tests',
path: File.expand_path('../../tmp', __dir__)
)
end
end

View file

@ -12,9 +12,7 @@ module QA
end
def logs
shell "docker logs #{@name}" do |line|
yield " #{line.chomp}"
end
shell "docker logs #{@name}"
end
def network

View file

@ -26,13 +26,13 @@ module QA
end
def config
@config ||= <<~END
@config ||= <<~CONFIG
concurrent = 1
check_interval = 0
[session_server]
session_timeout = 1800
END
CONFIG
end
def register!
@ -40,15 +40,14 @@ module QA
docker run -d --rm --network #{runner_network} --name #{@name}
#{'-v /var/run/docker.sock:/var/run/docker.sock' if @executor == :docker}
--privileged
#{@image} #{add_gitlab_tls_cert if @address.include? "https"} && docker exec --detach #{@name} sh -c "#{register_command}"
#{@image} #{add_gitlab_tls_cert if @address.include? 'https'}
&& docker exec --detach #{@name} sh -c "#{register_command}"
CMD
wait_until_running_and_configured
# Prove airgappedness
if runner_network == 'airgapped'
shell("docker exec #{@name} sh -c '#{prove_airgap}'")
end
shell("docker exec #{@name} sh -c '#{prove_airgap}'") if runner_network == 'airgapped'
end
def tags=(tags)
@ -66,7 +65,7 @@ module QA
args << "--registration-token #{@token}"
args << if run_untagged
raise CONFLICTING_VARIABLES_MESSAGE % [:tags=, :run_untagged, run_untagged] if @tags&.any?
raise format(CONFLICTING_VARIABLES_MESSAGE, :tags=, :run_untagged, run_untagged) if @tags&.any?
'--run-untagged=true'
else
@ -86,7 +85,7 @@ module QA
end
<<~CMD.strip
printf '#{config.chomp.gsub(/\n/, "\\n").gsub('"', '\"')}' > /etc/gitlab-runner/config.toml &&
printf '#{config.chomp.gsub(/\n/, '\\n').gsub('"', '\"')}' > /etc/gitlab-runner/config.toml &&
gitlab-runner register \
#{args.join(' ')} &&
gitlab-runner run

View file

@ -115,7 +115,7 @@ module QA
def node_state(name)
state = "stopped"
wait_until_shell_command("docker inspect -f {{.State.Status}} #{name}") do |line|
wait_until_shell_command("docker inspect -f {{.State.Status}} #{name}", stream_progress: false) do |line|
QA::Runtime::Logger.debug(line)
break state = "running" if line.include?("running")
break state = "paused" if line.include?("paused")
@ -164,7 +164,8 @@ module QA
end
def query_read_distribution
output = shell "docker exec #{@gitlab} bash -c 'curl -s http://localhost:9090/api/v1/query?query=gitaly_praefect_read_distribution'" do |line|
cmd = "docker exec #{@gitlab} bash -c 'curl -s http://localhost:9090/api/v1/query?query=gitaly_praefect_read_distribution'"
output = shell(cmd, stream_progress: false) do |line|
QA::Runtime::Logger.debug(line)
break line
end

View file

@ -6,34 +6,40 @@ module QA
module Service
module Shellout
using Rainbow
CommandError = Class.new(StandardError)
module_function
##
# TODO, make it possible to use generic QA framework classes
# as a library - gitlab-org/gitlab-qa#94
#
def shell(command, stdin_data: nil, fail_on_exception: true)
QA::Runtime::Logger.info("Executing `#{command}`".cyan)
def shell(command, stdin_data: nil, fail_on_exception: true, stream_progress: true) # rubocop:disable Metrics/CyclomaticComplexity
cmd_string = Array(command).join(' ')
QA::Runtime::Logger.info("Executing: `#{cmd_string.cyan}`")
Open3.popen2e(*command) do |stdin, out, wait|
stdin.puts(stdin_data) if stdin_data
stdin.close if stdin_data
print_progress_dots = stream_progress && !Runtime::Env.running_in_ci?
cmd_output = ''
if block_given?
out.each do |line|
cmd_output += line
yield line
end
out.each do |line|
cmd_output += line
yield line if block_given?
# indicate progress for local run by printing dots
print "." if print_progress_dots
end
out.each_char { |char| print char }
# add newline after progress dots
puts if print_progress_dots && !cmd_output.empty?
if wait.value.exited? && wait.value.exitstatus.nonzero? && fail_on_exception
raise CommandError, "Command failed: #{command} \nCommand Output: #{cmd_output}"
Runtime::Logger.error("Command output:\n#{cmd_output.strip}") unless cmd_output.empty?
raise CommandError, "Command: `#{cmd_string}` failed! ✘"
end
Runtime::Logger.debug("Command output:\n#{cmd_output.strip}") unless cmd_output.empty?
end
end
@ -46,18 +52,17 @@ module QA
def wait_until_shell_command(cmd, **kwargs)
sleep_interval = kwargs.delete(:sleep_interval) || 1
stream_progress = kwargs.delete(:stream_progress).then { |arg| arg.nil? ? true : false }
Support::Waiter.wait_until(sleep_interval: sleep_interval, **kwargs) do
shell cmd do |line|
shell(cmd, stream_progress: stream_progress) do |line|
break true if yield line
end
end
end
def wait_until_shell_command_matches(cmd, regex, **kwargs)
wait_until_shell_command(cmd, **kwargs) do |line|
QA::Runtime::Logger.debug(line.chomp)
wait_until_shell_command(cmd, stream_progress: false, **kwargs) do |line|
line =~ regex
end
end

View file

@ -53,7 +53,7 @@ module QA
def wait_and_check(actual, expectation_name)
attempt = 0
QA::Runtime::Logger.debug(
QA::Runtime::Logger.info(
"Running eventually matcher with '#{operator_msg}' operator with: '#{retry_args}' arguments"
)
QA::Support::Retrier.retry_until(**retry_args, log: false) do

View file

@ -13,15 +13,15 @@ module QA
end
def refresh(skip_finished_loading_check: false)
log("refreshing #{current_url}")
log("refreshing #{current_url}", :info)
super
end
def scroll_to(selector, text: nil)
msg = "scrolling to :#{Rainbow(selector).underline.bright}"
msg = "scrolling to :#{highlight_element(selector)}"
msg += " with text: #{text}" if text
log(msg)
log(msg, :info)
super
end
@ -39,7 +39,7 @@ module QA
element = super
log("found :#{Rainbow(name).underline.bright}")
log("found :#{name}")
element
end
@ -49,41 +49,41 @@ module QA
elements = super
log("found #{elements.size} :#{Rainbow(name).underline.bright}") if elements
log("found #{elements.size} :#{name}") if elements
elements
end
def check_element(name, click_by_js = nil)
log("checking :#{name}")
log("checking :#{highlight_element(name)}", :info)
super
end
def uncheck_element(name, click_by_js = nil)
log("unchecking :#{name}")
log("unchecking :#{highlight_element(name)}", :info)
super
end
def click_element_coordinates(name, **kwargs)
log(%Q(clicking the coordinates of :#{name}))
log(%(clicking the coordinates of :#{highlight_element(name)}), :info)
super
end
def click_element(name, page = nil, **kwargs)
msg = ["clicking :#{Rainbow(name).underline.bright}"]
msg = ["clicking :#{highlight_element(name)}"]
msg << ", expecting to be at #{page.class}" if page
msg << "with args #{kwargs}"
log(msg.compact.join(' '))
log(msg.join(' '), :info)
log("with args #{kwargs}")
super
end
def click_via_capybara(method, locator)
log("clicking via capybara using '#{method}(#{locator})'")
log("clicking via capybara using '#{method}(#{locator})'", :info)
super
end
@ -91,13 +91,13 @@ module QA
def fill_element(name, content)
masked_content = name.to_s.match?(/token|key|password/) ? '*****' : content
log(%Q(filling :#{name} with "#{masked_content}"))
log(%(filling :#{highlight_element(name)} with "#{masked_content}"), :info)
super
end
def select_element(name, value)
log(%Q(selecting "#{value}" in :#{name}))
log(%(selecting "#{value}" in :#{highlight_element(name)}), :info)
super
end
@ -121,7 +121,7 @@ module QA
def has_text?(text, **kwargs)
found = super
log(%Q{has_text?('#{text}', wait: #{kwargs[:wait] || Capybara.default_max_wait_time}) returned #{found}})
log(%(has_text?('#{text}', wait: #{kwargs[:wait] || Capybara.default_max_wait_time}) returned #{found}))
found
end
@ -129,7 +129,7 @@ module QA
def has_no_text?(text, **kwargs)
found = super
log(%Q{has_no_text?('#{text}', wait: #{kwargs[:wait] || Capybara.default_max_wait_time}) returned #{found}})
log(%(has_no_text?('#{text}', wait: #{kwargs[:wait] || Capybara.default_max_wait_time}) returned #{found}))
found
end
@ -173,13 +173,26 @@ module QA
private
def log(msg)
QA::Runtime::Logger.debug(msg)
# Log message
#
# @param [String] msg
# @param [Symbol] level
# @return [void]
def log(msg, level = :debug)
QA::Runtime::Logger.public_send(level, msg)
end
# Highlight element for enhanced logging
#
# @param [String] element
# @return [String]
def highlight_element(element)
element.to_s.underline.bright
end
def log_has_element_or_not(method, name, found, **kwargs)
msg = ["#{method} :#{Rainbow(name).underline.bright}"]
msg << %Q(with text "#{kwargs[:text]}") if kwargs[:text]
msg = ["#{method} :#{name}"]
msg << %(with text "#{kwargs[:text]}") if kwargs[:text]
msg << "class: #{kwargs[:class]}" if kwargs[:class]
msg << "(wait: #{kwargs[:wait] || Capybara.default_max_wait_time})"
msg << "returned: #{found}"

View file

@ -4,6 +4,7 @@ RSpec.describe QA::Git::Repository do
include QA::Support::Helpers::StubEnv
shared_context 'unresolvable git directory' do
let(:logger) { instance_double(Logger, info: nil, debug: nil) }
let(:repo_uri) { 'http://foo/bar.git' }
let(:repo_uri_with_credentials) { 'http://root@foo/bar.git' }
let(:env_vars) { [%q{HOME="temp"}] }
@ -22,6 +23,7 @@ RSpec.describe QA::Git::Repository do
before do
stub_env('GITLAB_USERNAME', 'root')
allow(repository).to receive(:tmp_home_dir).and_return(tmp_netrc_dir)
allow(QA::Runtime::Logger).to receive(:logger).and_return(logger)
end
around do |example|

View file

@ -112,7 +112,7 @@ RSpec.describe QA::Resource::Base do
let(:method) { 'api' }
before do
allow(QA::Runtime::Logger).to receive(:debug)
allow(QA::Runtime::Logger).to receive(:info)
allow(resource).to receive(:api_support?).and_return(true)
allow(resource).to receive(:fabricate_via_api!)
allow(resource).to receive(:api_client) { api_client }
@ -123,7 +123,7 @@ RSpec.describe QA::Resource::Base do
subject.fabricate_via_api!('something', resource: resource, parents: [])
expect(QA::Runtime::Logger).to have_received(:debug) do |&msg|
expect(QA::Runtime::Logger).to have_received(:info) do |&msg|
expect(msg.call).to match_regex(log_regex)
end
end
@ -155,7 +155,7 @@ RSpec.describe QA::Resource::Base do
let(:method) { 'browser_ui' }
before do
allow(QA::Runtime::Logger).to receive(:debug)
allow(QA::Runtime::Logger).to receive(:info)
end
it 'logs the resource and build method' do
@ -163,7 +163,7 @@ RSpec.describe QA::Resource::Base do
subject.fabricate_via_browser_ui!('something', resource: resource, parents: [])
expect(QA::Runtime::Logger).to have_received(:debug) do |&msg|
expect(QA::Runtime::Logger).to have_received(:info) do |&msg|
expect(msg.call).to match_regex(log_regex)
end
end

View file

@ -264,20 +264,6 @@ RSpec.describe QA::Runtime::Env do
end
end
describe '.log_destination' do
it 'returns $stdout if QA_LOG_PATH is not defined' do
stub_env('QA_LOG_PATH', nil)
expect(described_class.log_destination).to eq($stdout)
end
it 'returns the path if QA_LOG_PATH is defined' do
stub_env('QA_LOG_PATH', 'path/to_file')
expect(described_class.log_destination).to eq('path/to_file')
end
end
describe '.can_test?' do
it_behaves_like 'boolean method with parameter',
method: :can_test?,

View file

@ -2,6 +2,6 @@
RSpec.describe QA::Runtime::Logger do
it 'returns logger instance' do
expect(described_class.logger).to be_an_instance_of(::Logger)
expect(described_class.logger).to be_an_instance_of(ActiveSupport::Logger)
end
end

3
qa/tmp/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
*
!.gitignore