Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
19044caf66
commit
febc637ca9
|
|
@ -95,8 +95,6 @@ jsdoc/
|
|||
webpack-dev-server.json
|
||||
/.nvimrc
|
||||
.solargraph.yml
|
||||
/tmp/matching_foss_tests.txt
|
||||
/tmp/matching_tests.txt
|
||||
ee/changelogs/unreleased-ee
|
||||
/sitespeed-result
|
||||
tags.lock
|
||||
|
|
|
|||
|
|
@ -90,6 +90,8 @@ variables:
|
|||
RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json
|
||||
RSPEC_PROFILING_FOLDER_PATH: rspec/profiling
|
||||
FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json
|
||||
RSPEC_CHANGED_FILES_PATH: rspec/changed_files.txt
|
||||
RSPEC_MATCHING_TESTS_PATH: rspec/matching_tests.txt
|
||||
RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
|
||||
JUNIT_RESULT_FILE: rspec/junit_rspec.xml
|
||||
JUNIT_RETRY_FILE: rspec/junit_rspec-retry.xml
|
||||
|
|
|
|||
|
|
@ -8,11 +8,13 @@
|
|||
|
||||
.base-script:
|
||||
script:
|
||||
- source ./scripts/rspec_helpers.sh
|
||||
# Only install knapsack after bundle install! Otherwise oddly some native
|
||||
# gems could not be found under some circumstance. No idea why, hours wasted.
|
||||
- run_timed_command "gem install knapsack --no-document"
|
||||
- run_timed_command "scripts/gitaly-test-spawn"
|
||||
- source ./scripts/rspec_helpers.sh
|
||||
- echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-spawn[collapsed=true]\r\e[0KStarting Gitaly"
|
||||
- run_timed_command "scripts/gitaly-test-spawn" # Do not use 'bundle exec' here
|
||||
- echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-spawn\r\e[0K"
|
||||
|
||||
.minimal-rspec-tests:
|
||||
variables:
|
||||
|
|
@ -966,7 +968,7 @@ rspec fail-fast:
|
|||
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
|
||||
script:
|
||||
- !reference [.base-script, script]
|
||||
- rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
|
||||
- rspec_fail_fast "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine"
|
||||
artifacts:
|
||||
expire_in: 7d
|
||||
paths:
|
||||
|
|
@ -976,10 +978,10 @@ rspec foss-impact:
|
|||
extends:
|
||||
- .rspec-base-pg12-as-if-foss
|
||||
- .rails:rules:rspec-foss-impact
|
||||
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"]
|
||||
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"]
|
||||
script:
|
||||
- !reference [.base-script, script]
|
||||
- rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
|
||||
- rspec_matched_foss_tests "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine"
|
||||
artifacts:
|
||||
expire_in: 7d
|
||||
paths:
|
||||
|
|
|
|||
|
|
@ -110,10 +110,13 @@ generate-frontend-fixtures-mapping:
|
|||
paths:
|
||||
- ${FRONTEND_FIXTURES_MAPPING_PATH}
|
||||
|
||||
.detect-test-base:
|
||||
detect-tests:
|
||||
extends: .rails:rules:detect-tests
|
||||
image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
|
||||
needs: []
|
||||
stage: prepare
|
||||
variables:
|
||||
RSPEC_TESTS_MAPPING_ENABLED: "true"
|
||||
script:
|
||||
- source ./scripts/utils.sh
|
||||
- source ./scripts/rspec_helpers.sh
|
||||
|
|
@ -123,42 +126,23 @@ generate-frontend-fixtures-mapping:
|
|||
- retrieve_frontend_fixtures_mapping
|
||||
- |
|
||||
if [ -n "$CI_MERGE_REQUEST_IID" ]; then
|
||||
tooling/bin/find_changes ${CHANGES_FILE};
|
||||
tooling/bin/find_tests ${CHANGES_FILE} ${MATCHED_TESTS_FILE};
|
||||
tooling/bin/find_changes ${CHANGES_FILE} ${MATCHED_TESTS_FILE} ${FRONTEND_FIXTURES_MAPPING_PATH};
|
||||
echo "Changed files: $(cat $CHANGES_FILE)";
|
||||
echo "Related rspec tests: $(cat $MATCHED_TESTS_FILE)";
|
||||
mkdir -p $(dirname "$RSPEC_CHANGED_FILES_PATH")
|
||||
tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH};
|
||||
tooling/bin/find_tests ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH};
|
||||
tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH} ${FRONTEND_FIXTURES_MAPPING_PATH};
|
||||
echo "Changed files: $(cat $RSPEC_CHANGED_FILES_PATH)";
|
||||
echo "Related rspec tests: $(cat $RSPEC_MATCHING_TESTS_PATH)";
|
||||
fi
|
||||
artifacts:
|
||||
expire_in: 7d
|
||||
paths:
|
||||
- ${CHANGES_FILE}
|
||||
- ${MATCHED_TESTS_FILE}
|
||||
- ${RSPEC_CHANGED_FILES_PATH}
|
||||
- ${RSPEC_MATCHING_TESTS_PATH}
|
||||
- ${FRONTEND_FIXTURES_MAPPING_PATH}
|
||||
|
||||
detect-tests:
|
||||
extends:
|
||||
- .detect-test-base
|
||||
- .rails:rules:detect-tests
|
||||
variables:
|
||||
RSPEC_TESTS_MAPPING_ENABLED: "true"
|
||||
CHANGES_FILE: tmp/changed_files.txt
|
||||
MATCHED_TESTS_FILE: tmp/matching_tests.txt
|
||||
|
||||
detect-tests as-if-foss:
|
||||
extends:
|
||||
- .detect-test-base
|
||||
- .rails:rules:detect-tests
|
||||
- .as-if-foss
|
||||
variables:
|
||||
CHANGES_FILE: tmp/changed_foss_files.txt
|
||||
MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt
|
||||
before_script:
|
||||
- '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
|
||||
|
||||
detect-previous-failed-tests:
|
||||
extends:
|
||||
- .detect-test-base
|
||||
- detect-tests
|
||||
- .rails:rules:detect-previous-failed-tests
|
||||
variables:
|
||||
PREVIOUS_FAILED_TESTS_DIR: tmp/previous_failed_tests/
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ rubocop:
|
|||
if [ -z "${CI_MERGE_REQUEST_IID}" ] || [ "${RUN_ALL_RUBOCOP}" == "true" ]; then
|
||||
run_timed_command "bundle exec rubocop --parallel"
|
||||
else
|
||||
run_timed_command "bundle exec rubocop --parallel --force-exclusion $(cat tmp/changed_files.txt)"
|
||||
run_timed_command "bundle exec rubocop --parallel --force-exclusion $(cat ${RSPEC_CHANGED_FILES_PATH})"
|
||||
fi
|
||||
|
||||
qa:metadata-lint:
|
||||
|
|
|
|||
|
|
@ -67,9 +67,9 @@ module Projects
|
|||
end
|
||||
|
||||
def remove_snippets
|
||||
# We're setting the hard_delete param because we dont need to perform the access checks within the service since
|
||||
# We're setting the skip_authorization param because we dont need to perform the access checks within the service since
|
||||
# the user has enough access rights to remove the project and its resources.
|
||||
response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(hard_delete: true)
|
||||
response = ::Snippets::BulkDestroyService.new(current_user, project.snippets).execute(skip_authorization: true)
|
||||
|
||||
if response.error?
|
||||
log_error("Snippet deletion failed on #{project.full_path} with the following message: #{response.message}")
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ module Snippets
|
|||
@snippets = snippets
|
||||
end
|
||||
|
||||
def execute(options = {})
|
||||
def execute(skip_authorization: false)
|
||||
return ServiceResponse.success(message: 'No snippets found.') if snippets.empty?
|
||||
|
||||
user_can_delete_snippets! unless options[:hard_delete]
|
||||
user_can_delete_snippets! unless skip_authorization
|
||||
attempt_delete_repositories!
|
||||
snippets.destroy_all # rubocop: disable Cop/DestroyAll
|
||||
|
||||
|
|
|
|||
|
|
@ -58,7 +58,9 @@ module Users
|
|||
|
||||
MigrateToGhostUserService.new(user).execute(hard_delete: options[:hard_delete])
|
||||
|
||||
response = Snippets::BulkDestroyService.new(current_user, user.snippets).execute(options)
|
||||
skip_authorization = options.fetch(:hard_delete, false)
|
||||
response = Snippets::BulkDestroyService.new(current_user, user.snippets)
|
||||
.execute(skip_authorization: skip_authorization)
|
||||
raise DestroyError, response.message if response.error?
|
||||
|
||||
# Rails attempts to load all related records into memory before
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ table_name: spam_logs
|
|||
classes:
|
||||
- SpamLog
|
||||
feature_categories:
|
||||
- authentication_and_authorization
|
||||
description: TODO
|
||||
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d20e75a8d80c2828336cd22897ea6868d666f8a5
|
||||
- instance_resiliency
|
||||
description: Logs users flagged by the Akismet anti-spam integration.
|
||||
introduced_by_url: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2266
|
||||
milestone: '8.5'
|
||||
|
|
|
|||
|
|
@ -1143,14 +1143,16 @@ To delete an on-demand scan:
|
|||
1. In the saved scan's row select **More actions** (**{ellipsis_v}**), then select **Delete**.
|
||||
1. Select **Delete** to confirm the deletion.
|
||||
|
||||
### Site profile
|
||||
## Site profile
|
||||
|
||||
A site profile describes the attributes of a web site to scan on demand with DAST. A site profile is
|
||||
required for an on-demand DAST scan.
|
||||
A site profile defines the attributes and configuration details of the deployed application,
|
||||
website, or API to be scanned by DAST. A site profile can be referenced in `.gitlab-ci.yml` and
|
||||
on-demand scans.
|
||||
|
||||
A site profile contains the following:
|
||||
A site profile contains:
|
||||
|
||||
- **Profile name**: A name you assign to the site to be scanned.
|
||||
- **Profile name**: A name you assign to the site to be scanned. While a site profile is referenced
|
||||
in either `.gitlab-ci.yml` or an on-demand scan, it **cannot** be renamed.
|
||||
- **Site type**: The type of target to be scanned, either website or API scan.
|
||||
- **Target URL**: The URL that DAST runs against.
|
||||
- **Excluded URLs**: A comma-separated list of URLs to exclude from the scan.
|
||||
|
|
@ -1168,7 +1170,7 @@ When an API site type is selected, a [host override](#host-override) is used to
|
|||
When configured, request headers and password fields are encrypted using [`aes-256-gcm`](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) before being stored in the database.
|
||||
This data can only be read and decrypted with a valid secrets file.
|
||||
|
||||
#### Site profile validation
|
||||
### Site profile validation
|
||||
|
||||
> - Site profile validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233020) in GitLab 13.8.
|
||||
> - Meta tag validation [introduced](https://gitlab.com/groups/gitlab-org/-/epics/6460) in GitLab 14.2.
|
||||
|
|
@ -1192,7 +1194,7 @@ All these methods are equivalent in functionality. Use whichever is feasible.
|
|||
In [GitLab 14.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/324990), site profile
|
||||
validation happens in a CI job using the [GitLab Runner](../../../ci/runners/index.md).
|
||||
|
||||
#### Create a site profile
|
||||
### Create a site profile
|
||||
|
||||
To create a site profile:
|
||||
|
||||
|
|
@ -1203,7 +1205,7 @@ To create a site profile:
|
|||
|
||||
The site profile is created.
|
||||
|
||||
#### Edit a site profile
|
||||
### Edit a site profile
|
||||
|
||||
If a site profile is linked to a security policy, a user cannot edit the profile from this page. See
|
||||
[Scan execution policies](../policies/scan-execution-policies.md)
|
||||
|
|
@ -1220,7 +1222,7 @@ To edit a site profile:
|
|||
1. In the profile's row select the **More actions** (**{ellipsis_v}**) menu, then select **Edit**.
|
||||
1. Edit the fields then select **Save profile**.
|
||||
|
||||
#### Delete a site profile
|
||||
### Delete a site profile
|
||||
|
||||
If a site profile is linked to a security policy, a user cannot delete the profile from this page.
|
||||
See [Scan execution policies](../policies/scan-execution-policies.md)
|
||||
|
|
@ -1234,7 +1236,7 @@ To delete a site profile:
|
|||
1. In the profile's row, select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**.
|
||||
1. Select **Delete** to confirm the deletion.
|
||||
|
||||
#### Validate a site profile
|
||||
### Validate a site profile
|
||||
|
||||
Validating a site is required to run an active scan.
|
||||
|
||||
|
|
@ -1266,7 +1268,7 @@ To validate a site profile:
|
|||
The site is validated and an active scan can run against it. A site profile's validation status is
|
||||
revoked only when it's revoked manually, or its file, header, or meta tag is edited.
|
||||
|
||||
#### Retry a failed validation
|
||||
### Retry a failed validation
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322609) in GitLab 14.3.
|
||||
> - [Deployed behind the `dast_failed_site_validations` flag](../../../administration/feature_flags.md), enabled by default.
|
||||
|
|
@ -1283,7 +1285,7 @@ To retry a site profile's failed validation:
|
|||
1. Select the **Site Profiles** tab.
|
||||
1. In the profile's row, select **Retry validation**.
|
||||
|
||||
#### Revoke a site profile's validation status
|
||||
### Revoke a site profile's validation status
|
||||
|
||||
WARNING:
|
||||
When a site profile's validation status is revoked, all site profiles that share the same URL also
|
||||
|
|
@ -1297,12 +1299,12 @@ To revoke a site profile's validation status:
|
|||
|
||||
The site profile's validation status is revoked.
|
||||
|
||||
#### Validated site profile headers
|
||||
### Validated site profile headers
|
||||
|
||||
The following are code samples of how you can provide the required site profile header in your
|
||||
application.
|
||||
|
||||
##### Ruby on Rails example for on-demand scan
|
||||
#### Ruby on Rails example for on-demand scan
|
||||
|
||||
Here's how you can add a custom header in a Ruby on Rails application:
|
||||
|
||||
|
|
@ -1315,7 +1317,7 @@ class DastWebsiteTargetController < ActionController::Base
|
|||
end
|
||||
```
|
||||
|
||||
##### Django example for on-demand scan
|
||||
#### Django example for on-demand scan
|
||||
|
||||
Here's how you can add a
|
||||
[custom header in Django](https://docs.djangoproject.com/en/2.2/ref/request-response/#setting-header-fields):
|
||||
|
|
@ -1329,7 +1331,7 @@ class DastWebsiteTargetView(View):
|
|||
return response
|
||||
```
|
||||
|
||||
##### Node (with Express) example for on-demand scan
|
||||
#### Node (with Express) example for on-demand scan
|
||||
|
||||
Here's how you can add a
|
||||
[custom header in Node (with Express)](https://expressjs.com/en/5x/api.html#res.append):
|
||||
|
|
@ -1341,22 +1343,26 @@ app.get('/dast-website-target', function(req, res) {
|
|||
})
|
||||
```
|
||||
|
||||
### Scanner profile
|
||||
## Scanner profile
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222767) in GitLab 13.4.
|
||||
> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/225804) in GitLab 13.5: scan mode, AJAX spider, debug messages.
|
||||
|
||||
A scanner profile defines the scanner settings used to run an on-demand scan:
|
||||
A scanner profile defines the configuration details of a security scanner. A scanner profile can be
|
||||
referenced in `.gitlab-ci.yml` and on-demand scans.
|
||||
|
||||
- **Profile name:** A name you give the scanner profile. For example, "Spider_15".
|
||||
A scanner profile contains:
|
||||
|
||||
- **Profile name:** A name you give the scanner profile. For example, "Spider_15". While a scanner
|
||||
profile is referenced in either `.gitlab-ci.yml` or an on-demand scan, it **cannot** be renamed.
|
||||
- **Scan mode:** A passive scan monitors all HTTP messages (requests and responses) sent to the target. An active scan attacks the target to find potential vulnerabilities.
|
||||
- **Spider timeout:** The maximum number of minutes allowed for the spider to traverse the site.
|
||||
- **Target timeout:** The maximum number of seconds DAST waits for the site to be available before
|
||||
starting the scan.
|
||||
- **AJAX spider:** Run the AJAX spider, in addition to the traditional spider, to crawl the target site.
|
||||
- **AJAX spider:** Run the AJAX spider, in addition to the traditional spider, to crawl the target site.
|
||||
- **Debug messages:** Include debug messages in the DAST console output.
|
||||
|
||||
#### Create a scanner profile
|
||||
### Create a scanner profile
|
||||
|
||||
To create a scanner profile:
|
||||
|
||||
|
|
@ -1366,7 +1372,7 @@ To create a scanner profile:
|
|||
1. Complete the form. For details of each field, see [Scanner profile](#scanner-profile).
|
||||
1. Select **Save profile**.
|
||||
|
||||
#### Edit a scanner profile
|
||||
### Edit a scanner profile
|
||||
|
||||
If a scanner profile is linked to a security policy, a user cannot edit the profile from this page.
|
||||
See [Scan execution policies](../policies/scan-execution-policies.md)
|
||||
|
|
@ -1381,7 +1387,7 @@ To edit a scanner profile:
|
|||
1. Edit the form.
|
||||
1. Select **Save profile**.
|
||||
|
||||
#### Delete a scanner profile
|
||||
### Delete a scanner profile
|
||||
|
||||
If a scanner profile is linked to a security policy, a user cannot delete the profile from this
|
||||
page. See [Scan execution policies](../policies/scan-execution-policies.md)
|
||||
|
|
|
|||
|
|
@ -49,8 +49,11 @@ To reduce false negatives in [dependency scans](../../../user/application_securi
|
|||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285477) in GitLab 13.11, date range slider to visualize data between given dates.
|
||||
|
||||
The project Security Dashboard shows the total number of vulnerabilities
|
||||
over time, with up to 365 days of historical data. Data refreshes daily at 01:15 UTC.
|
||||
It shows statistics for all vulnerabilities.
|
||||
over time, with up to 365 days of historical data. Data refresh begins daily at 01:15 UTC via a scheduled job.
|
||||
Each refresh captures a snapshot of open vulnerabilities. Data is not backported to prior days
|
||||
so vulnerabilities opened after the job has already run for the day will not be reflected in the
|
||||
counts until the following day's refresh job.
|
||||
Project Security Dashboards show statistics for all vulnerabilities with a current status of `Needs triage` or `Confirmed` .
|
||||
|
||||
To view total number of vulnerabilities over time:
|
||||
|
||||
|
|
|
|||
|
|
@ -83,11 +83,11 @@ incorporated once the pipeline finishes.
|
|||
|
||||
When a pipeline contains jobs that produce multiple security reports of the same type, it is possible that the same
|
||||
vulnerability finding is present in multiple reports. This duplication is common when different scanners are used to
|
||||
increase coverage. The deduplication process allows you to maximize the vulnerability scanning coverage while reducing
|
||||
increase coverage, but can also exist within a single report. The deduplication process allows you to maximize the vulnerability scanning coverage while reducing
|
||||
the number of findings you need to manage.
|
||||
|
||||
A finding is considered a duplicate of another finding when their [scan type](../terminology/index.md#scan-type-report-type),
|
||||
[location](../terminology/index.md#location-fingerprint) and
|
||||
[location](../terminology/index.md#location-fingerprint), and one or more of its
|
||||
[identifiers](../../../development/integrations/secure.md#identifiers) are the same.
|
||||
|
||||
The scan type must match because each can have its own definition for the location of a vulnerability. For example,
|
||||
|
|
@ -95,8 +95,9 @@ static analyzers are able to locate a file path and line number, whereas a conta
|
|||
name instead.
|
||||
|
||||
When comparing identifiers, GitLab does not compare `CWE` and `WASC` during deduplication because they are
|
||||
"type identifiers" and are used to classify groups of vulnerabilities. Including these identifiers results in
|
||||
many findings being incorrectly considered duplicates.
|
||||
"type identifiers" and are used to classify groups of vulnerabilities. Including these identifiers would result in
|
||||
many findings being incorrectly considered duplicates. Two findings are considered unique if none of their
|
||||
identifiers match.
|
||||
|
||||
In a set of duplicated findings, the first occurrence of a finding is kept and the remaining are skipped. Security
|
||||
reports are processed in alphabetical file path order, and findings are processed sequentially in the order they
|
||||
|
|
@ -124,16 +125,17 @@ appear in a report.
|
|||
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
|
||||
- Identifiers: CWE-798
|
||||
- Deduplication result: duplicates because `CWE` identifiers are ignored.
|
||||
- Example 3: matching scan type, location and identifiers.
|
||||
- Example 3: matching scan type, location and an identifier.
|
||||
- Finding
|
||||
- Scan type: `container_scanning`
|
||||
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
|
||||
- Identifiers: CVE-2022-25510, CWE-259
|
||||
- Identifiers: CVE-2019-12345, CVE-2022-25510, CWE-259
|
||||
- Other Finding
|
||||
- Scan type: `container_scanning`
|
||||
- Location fingerprint: `adc83b19e793491b1c6ea0fd8b46cd9f32e592fc`
|
||||
- Identifiers: CVE-2022-25510, CWE-798
|
||||
- Deduplication result: duplicates because all criteria match, and type identifiers are ignored.
|
||||
Only one identifier needs to match, in this case CVE-2022-25510.
|
||||
|
||||
The examples above don't include the raw location values. Each scan type defines its own
|
||||
`fingerprint_data`, which is used to generate a `SHA1` hash that is used as the `location_fingerprint`.
|
||||
|
|
|
|||
|
|
@ -11,8 +11,7 @@ module API
|
|||
optional :visibility, type: String,
|
||||
values: Gitlab::VisibilityLevel.string_values,
|
||||
desc: 'The visibility of the group'
|
||||
# TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
|
||||
optional :avatar, type: File, desc: 'Avatar image for the group' # rubocop:disable Scalability/FileUploads
|
||||
optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for the group'
|
||||
optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
|
||||
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication'
|
||||
optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced'
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ module Gitlab
|
|||
)
|
||||
end
|
||||
|
||||
relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class)
|
||||
next_batch_bounds = nil
|
||||
|
||||
relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop
|
||||
|
|
@ -44,15 +43,6 @@ module Gitlab
|
|||
next_batch_bounds
|
||||
end
|
||||
|
||||
# Deprecated
|
||||
#
|
||||
# Use `scope_to` to define additional filters on the migration job class.
|
||||
#
|
||||
# see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters.
|
||||
def apply_additional_filters(relation, job_arguments: [], job_class: nil)
|
||||
relation
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: [])
|
||||
|
|
|
|||
|
|
@ -3,23 +3,9 @@
|
|||
module Gitlab
|
||||
module BackgroundMigration
|
||||
module BatchingStrategies
|
||||
# Batching class to use for removing backfilled job artifact expire_at.
|
||||
# Batches will be scoped to records where either:
|
||||
# - expire_at is set to midnight on the 22nd of the month of the local timezone,
|
||||
# - record that has file_type = 3 (trace)
|
||||
#
|
||||
# If no more batches exist in the table, returns nil.
|
||||
# Used to apply additional filters to the batching table, migrated to
|
||||
# use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478
|
||||
class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy
|
||||
EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
|
||||
EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
|
||||
AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
|
||||
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
|
||||
SQL
|
||||
|
||||
def apply_additional_filters(relation, job_arguments: [], job_class: nil)
|
||||
relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
|
||||
.or(relation.where(file_type: 3))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -26,13 +26,16 @@ module Gitlab
|
|||
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
|
||||
SQL
|
||||
|
||||
scope_to ->(relation) {
|
||||
relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
|
||||
.or(relation.where(file_type: 3))
|
||||
}
|
||||
|
||||
def perform
|
||||
each_sub_batch(
|
||||
operation_name: :update_all
|
||||
) do |sub_batch|
|
||||
sub_batch.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
|
||||
.or(sub_batch.where(file_type: 3))
|
||||
.update_all(expire_at: nil)
|
||||
sub_batch.update_all(expire_at: nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
"jest": "jest --config jest.config.js",
|
||||
"jest-debug": "node --inspect-brk node_modules/.bin/jest --runInBand",
|
||||
"jest:ci": "jest --config jest.config.js --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
|
||||
"jest:ci:minimal": "jest --config jest.config.js --ci --coverage --findRelatedTests $(cat tmp/changed_files.txt) --passWithNoTests --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
|
||||
"jest:ci:minimal": "jest --config jest.config.js --ci --coverage --findRelatedTests $(cat $RSPEC_MATCHING_TESTS_PATH) --passWithNoTests --testSequencer ./scripts/frontend/parallel_ci_sequencer.js",
|
||||
"jest:integration": "jest --config jest.config.integration.js",
|
||||
"lint:eslint": "node scripts/frontend/eslint.js",
|
||||
"lint:eslint:fix": "node scripts/frontend/eslint.js --fix",
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ function rspec_paralellized_job() {
|
|||
debug_rspec_variables
|
||||
|
||||
if [[ -n $RSPEC_TESTS_MAPPING_ENABLED ]]; then
|
||||
tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" --filter "tmp/matching_tests.txt" || rspec_run_status=$?
|
||||
tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" --filter "${RSPEC_MATCHING_TESTS_PATH}" || rspec_run_status=$?
|
||||
else
|
||||
tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" || rspec_run_status=$?
|
||||
fi
|
||||
|
|
@ -360,9 +360,22 @@ function rspec_fail_fast() {
|
|||
function rspec_matched_foss_tests() {
|
||||
local test_file_count_threshold=20
|
||||
local matching_tests_file=${1}
|
||||
local foss_matching_tests_file="${matching_tests_file}-foss"
|
||||
|
||||
# Keep only files that exists (i.e. exclude EE speficic files)
|
||||
cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| File.exist?(f) && f.include?("spec/") }.join(" ")' > "${foss_matching_tests_file}"
|
||||
|
||||
echo "Matching tests file:"
|
||||
cat ${matching_tests_file}
|
||||
echo -e "\n\n"
|
||||
|
||||
echo "FOSS matching tests file:"
|
||||
cat ${foss_matching_tests_file}
|
||||
echo -e "\n\n"
|
||||
|
||||
local rspec_opts=${2}
|
||||
local test_files="$(cat "${matching_tests_file}")"
|
||||
local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
|
||||
local test_files="$(cat ${foss_matching_tests_file})"
|
||||
local test_file_count=$(wc -w "${foss_matching_tests_file}" | awk {'print $1'})
|
||||
|
||||
if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
|
||||
echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched,"
|
||||
|
|
|
|||
|
|
@ -77,25 +77,4 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'additional filters' do
|
||||
let(:strategy_with_filters) do
|
||||
Class.new(described_class) do
|
||||
def apply_additional_filters(relation, job_arguments:, job_class: nil)
|
||||
min_id = job_arguments.first
|
||||
|
||||
relation.where.not(type: 'Project').where('id >= ?', min_id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
let(:batching_strategy) { strategy_with_filters.new(connection: ActiveRecord::Base.connection) }
|
||||
let!(:namespace5) { namespaces.create!(name: 'batchtest5', path: 'batch-test5', type: 'Project') }
|
||||
|
||||
it 'applies additional filters' do
|
||||
batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [1])
|
||||
|
||||
expect(batch_bounds).to eq([namespace4.id, namespace4.id])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,100 +2,6 @@
|
|||
|
||||
require 'spec_helper'
|
||||
|
||||
RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength
|
||||
let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
|
||||
let_it_be(:project) do
|
||||
table(:projects).create!(
|
||||
id: 1,
|
||||
name: 'gitlab1',
|
||||
path: 'gitlab1',
|
||||
project_namespace_id: 1,
|
||||
namespace_id: namespace.id
|
||||
)
|
||||
end
|
||||
|
||||
let(:batching_strategy) { described_class.new(connection: Ci::ApplicationRecord.connection) }
|
||||
let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
|
||||
|
||||
# job artifacts expiring at midnight in various timezones
|
||||
let!(:ci_job_artifact_1) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_2) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) }
|
||||
let!(:ci_job_artifact_3) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) }
|
||||
let!(:ci_job_artifact_4) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) }
|
||||
let!(:ci_job_artifact_5) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) }
|
||||
let!(:ci_job_artifact_6) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) }
|
||||
let!(:ci_job_artifact_7) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) }
|
||||
# out ot scope job artifacts
|
||||
let!(:ci_job_artifact_8) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) }
|
||||
let!(:ci_job_artifact_9) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) }
|
||||
# job artifacts of trace type (file_type: 3)
|
||||
let!(:ci_job_artifact_10) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_11) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
# out ot scope job artifacts
|
||||
let!(:ci_job_artifact_12) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) }
|
||||
let!(:ci_job_artifact_13) { create_job_artifact(file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) }
|
||||
# job artifacts of trace type (file_type: 3)
|
||||
let!(:ci_job_artifact_14) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) }
|
||||
let!(:ci_job_artifact_15) { create_job_artifact(file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) }
|
||||
|
||||
RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy do # rubocop:disable Layout/LineLength
|
||||
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
|
||||
|
||||
context 'when starting on the first batch' do
|
||||
it 'returns the bounds of the next batch' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_1.id,
|
||||
batch_size: 5,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_5.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the range includes out of scope records' do
|
||||
it 'returns the bounds of the next batch, skipping records outside the scope' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_1.id,
|
||||
batch_size: 10,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_1.id, ci_job_artifact_14.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when the range begins on out of scope records' do
|
||||
it 'returns the bounds of the next batch, skipping records outside the scope' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_8.id,
|
||||
batch_size: 3,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to eq([ci_job_artifact_10.id, ci_job_artifact_14.id])
|
||||
end
|
||||
end
|
||||
|
||||
context 'when no additional batch remain' do
|
||||
it 'returns nil' do
|
||||
batch_bounds = batching_strategy.next_batch(
|
||||
:ci_job_artifacts,
|
||||
:id,
|
||||
batch_min_value: ci_job_artifact_15.id + 1,
|
||||
batch_size: 10,
|
||||
job_arguments: []
|
||||
)
|
||||
expect(batch_bounds).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def create_job_artifact(file_type:, expire_at:)
|
||||
job = table(:ci_builds, database: :ci).create!
|
||||
job_artifact.create!(job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ require 'spec_helper'
|
|||
RSpec.describe API::Groups do
|
||||
include GroupAPIHelpers
|
||||
include UploadHelpers
|
||||
include WorkhorseHelpers
|
||||
|
||||
let_it_be(:user1) { create(:user, can_create_group: false) }
|
||||
let_it_be(:user2) { create(:user) }
|
||||
|
|
@ -872,21 +873,31 @@ RSpec.describe API::Groups do
|
|||
group_param = {
|
||||
avatar: fixture_file_upload(file_path)
|
||||
}
|
||||
put api("/groups/#{group1.id}", user1), params: group_param
|
||||
workhorse_form_with_file(
|
||||
api("/groups/#{group1.id}", user1),
|
||||
method: :put,
|
||||
file_key: :avatar,
|
||||
params: group_param
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when authenticated as the group owner' do
|
||||
it 'updates the group' do
|
||||
put api("/groups/#{group1.id}", user1), params: {
|
||||
name: new_group_name,
|
||||
request_access_enabled: true,
|
||||
project_creation_level: "noone",
|
||||
subgroup_creation_level: "maintainer",
|
||||
default_branch_protection: ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS,
|
||||
prevent_sharing_groups_outside_hierarchy: true,
|
||||
avatar: fixture_file_upload(file_path)
|
||||
}
|
||||
workhorse_form_with_file(
|
||||
api("/groups/#{group1.id}", user1),
|
||||
method: :put,
|
||||
file_key: :avatar,
|
||||
params: {
|
||||
name: new_group_name,
|
||||
request_access_enabled: true,
|
||||
project_creation_level: "noone",
|
||||
subgroup_creation_level: "maintainer",
|
||||
default_branch_protection: ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS,
|
||||
prevent_sharing_groups_outside_hierarchy: true,
|
||||
avatar: fixture_file_upload(file_path)
|
||||
}
|
||||
)
|
||||
|
||||
expect(response).to have_gitlab_http_status(:ok)
|
||||
expect(json_response['name']).to eq(new_group_name)
|
||||
|
|
@ -1787,7 +1798,12 @@ RSpec.describe API::Groups do
|
|||
attrs[:avatar] = fixture_file_upload(file_path)
|
||||
end
|
||||
|
||||
post api("/groups", user3), params: params
|
||||
workhorse_form_with_file(
|
||||
api('/groups', user3),
|
||||
method: :post,
|
||||
file_key: :avatar,
|
||||
params: params
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -423,11 +423,11 @@ RSpec.describe Projects::DestroyService, :aggregate_failures, :event_store_publi
|
|||
destroy_project(project, user)
|
||||
end
|
||||
|
||||
it 'calls the bulk snippet destroy service with the hard_delete param set to true' do
|
||||
it 'calls the bulk snippet destroy service with the skip_authorization param set to true' do
|
||||
expect(project.snippets.count).to eq 2
|
||||
|
||||
expect_next_instance_of(Snippets::BulkDestroyService, user, project.snippets) do |instance|
|
||||
expect(instance).to receive(:execute).with(hard_delete: true).and_call_original
|
||||
expect(instance).to receive(:execute).with(skip_authorization: true).and_call_original
|
||||
end
|
||||
|
||||
expect do
|
||||
|
|
@ -485,9 +485,11 @@ RSpec.describe Projects::DestroyService, :aggregate_failures, :event_store_publi
|
|||
let!(:project_bot) { create(:user, :project_bot).tap { |user| project.add_maintainer(user) } }
|
||||
|
||||
it 'deletes bot user as well' do
|
||||
expect do
|
||||
destroy_project(project, user)
|
||||
end.to change { User.find_by(id: project_bot.id) }.to(nil)
|
||||
expect_next_instance_of(Users::DestroyService, user) do |instance|
|
||||
expect(instance).to receive(:execute).with(project_bot, skip_authorization: true).and_call_original
|
||||
end
|
||||
|
||||
destroy_project(project, user)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -71,8 +71,8 @@ RSpec.describe Snippets::BulkDestroyService do
|
|||
let(:error_message) { "You don't have access to delete these snippets." }
|
||||
end
|
||||
|
||||
context 'when hard_delete option is passed' do
|
||||
subject { described_class.new(service_user, snippets).execute(hard_delete: true) }
|
||||
context 'when skip_authorization option is passed' do
|
||||
subject { described_class.new(service_user, snippets).execute(skip_authorization: true) }
|
||||
|
||||
it 'returns a ServiceResponse success response' do
|
||||
expect(subject).to be_success
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ RSpec.describe Users::DestroyService do
|
|||
allow(user).to receive(:personal_projects).and_return([])
|
||||
|
||||
expect_next_instance_of(Snippets::BulkDestroyService) do |bulk_destroy_service|
|
||||
expect(bulk_destroy_service).to receive(:execute).with({ hard_delete: true }).and_call_original
|
||||
expect(bulk_destroy_service).to receive(:execute).with({ skip_authorization: true }).and_call_original
|
||||
end
|
||||
|
||||
service.execute(user, { hard_delete: true })
|
||||
|
|
|
|||
|
|
@ -329,6 +329,10 @@ func configureRoutes(u *upstream) {
|
|||
u.route("POST", apiPattern+`v4/projects\z`, tempfileMultipartProxy),
|
||||
u.route("PUT", apiProjectPattern+`\z`, tempfileMultipartProxy),
|
||||
|
||||
// Group Avatar
|
||||
u.route("POST", apiPattern+`v4/groups\z`, tempfileMultipartProxy),
|
||||
u.route("PUT", apiPattern+`v4/groups/[^/]+\z`, tempfileMultipartProxy),
|
||||
|
||||
// Explicitly proxy API requests
|
||||
u.route("", apiPattern, proxy),
|
||||
u.route("", ciAPIPattern, proxy),
|
||||
|
|
|
|||
|
|
@ -135,6 +135,9 @@ func TestAcceleratedUpload(t *testing.T) {
|
|||
{"POST", `/api/graphql`, false},
|
||||
{"POST", `/api/v4/topics`, false},
|
||||
{"PUT", `/api/v4/topics`, false},
|
||||
{"POST", `/api/v4/groups`, false},
|
||||
{"PUT", `/api/v4/groups/5`, false},
|
||||
{"PUT", `/api/v4/groups/group%2Fsubgroup`, false},
|
||||
{"PUT", "/api/v4/projects/9001/packages/nuget/v1/files", true},
|
||||
{"PUT", "/api/v4/projects/group%2Fproject/packages/nuget/v1/files", true},
|
||||
{"PUT", "/api/v4/projects/group%2Fsubgroup%2Fproject/packages/nuget/v1/files", true},
|
||||
|
|
|
|||
Loading…
Reference in New Issue