diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index e52986b7fcc..65800e40d6c 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -529,6 +529,7 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :akismet_api_key, encryption_options_base_32_aes_256_gcm attr_encrypted :spam_check_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false) attr_encrypted :elasticsearch_aws_secret_access_key, encryption_options_base_32_aes_256_gcm + attr_encrypted :elasticsearch_password, encryption_options_base_32_aes_256_gcm.merge(encode: false) attr_encrypted :recaptcha_private_key, encryption_options_base_32_aes_256_gcm attr_encrypted :recaptcha_site_key, encryption_options_base_32_aes_256_gcm attr_encrypted :slack_app_secret, encryption_options_base_32_aes_256_gcm diff --git a/changelogs/unreleased/328466-add-username-password-to-es-settings.yml b/changelogs/unreleased/328466-add-username-password-to-es-settings.yml new file mode 100644 index 00000000000..fd2ec65ccf5 --- /dev/null +++ b/changelogs/unreleased/328466-add-username-password-to-es-settings.yml @@ -0,0 +1,5 @@ +--- +title: Add username and password fields for Advanced Search +merge_request: 60710 +author: +type: changed diff --git a/config/application.rb b/config/application.rb index 320cfbf824f..6002b668bba 100644 --- a/config/application.rb +++ b/config/application.rb @@ -84,6 +84,7 @@ module Gitlab # Rake tasks ignore the eager loading settings, so we need to set the # autoload paths explicitly config.autoload_paths = config.eager_load_paths.dup + config.autoload_paths.push("#{config.root}/lib/generators") # Only load the plugins named here, in the order given (default is alphabetical). # :all can be used as a placeholder for all plugins not explicitly named. @@ -145,6 +146,7 @@ module Gitlab encrypted_key import_url elasticsearch_url + elasticsearch_password search jwt otp_attempt diff --git a/db/migrate/20210430100947_add_elasticsearch_username_password_to_application_settings.rb b/db/migrate/20210430100947_add_elasticsearch_username_password_to_application_settings.rb new file mode 100644 index 00000000000..446ac93c75b --- /dev/null +++ b/db/migrate/20210430100947_add_elasticsearch_username_password_to_application_settings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddElasticsearchUsernamePasswordToApplicationSettings < ActiveRecord::Migration[6.0] + def change + # rubocop:disable Migration/AddLimitToTextColumns + # limit is added in 20210505124816_add_text_limit_to_elasticsearch_username + add_column :application_settings, :elasticsearch_username, :text + # rubocop:enable Migration/AddLimitToTextColumns + + add_column :application_settings, :encrypted_elasticsearch_password, :binary + add_column :application_settings, :encrypted_elasticsearch_password_iv, :binary + end +end diff --git a/db/migrate/20210505124816_add_text_limit_to_elasticsearch_username.rb b/db/migrate/20210505124816_add_text_limit_to_elasticsearch_username.rb new file mode 100644 index 00000000000..324b9193efb --- /dev/null +++ b/db/migrate/20210505124816_add_text_limit_to_elasticsearch_username.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddTextLimitToElasticsearchUsername < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_text_limit :application_settings, :elasticsearch_username, 255 + end + + def down + remove_text_limit :application_settings, :elasticsearch_username + end +end diff --git a/db/schema_migrations/20210430100947 b/db/schema_migrations/20210430100947 new file mode 100644 index 00000000000..409d807ff2a --- /dev/null +++ b/db/schema_migrations/20210430100947 @@ -0,0 +1 @@ +c8875e02134542370cc5a792bdaefc77b66f58a33a46720f5ef562c33c5b8f41 \ No newline at end of file diff --git a/db/schema_migrations/20210505124816 b/db/schema_migrations/20210505124816 new file mode 100644 index 00000000000..a5877ffee20 --- /dev/null +++ b/db/schema_migrations/20210505124816 @@ -0,0 +1 @@ +56aa9590f4bc37d8f8c4ed869a4b095ba39925fb06ab58500eead895d19ee336 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index eaab32aa66f..efc5811c52e 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9514,6 +9514,9 @@ CREATE TABLE application_settings ( encrypted_spam_check_api_key bytea, encrypted_spam_check_api_key_iv bytea, floc_enabled boolean DEFAULT false NOT NULL, + elasticsearch_username text, + encrypted_elasticsearch_password bytea, + encrypted_elasticsearch_password_iv bytea, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)), CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)), @@ -9529,6 +9532,7 @@ CREATE TABLE application_settings ( CONSTRAINT check_a5704163cc CHECK ((char_length(secret_detection_revocation_token_types_url) <= 255)), CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)), CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)), + CONSTRAINT check_e5024c8801 CHECK ((char_length(elasticsearch_username) <= 255)), CONSTRAINT check_e5aba18f02 CHECK ((char_length(container_registry_version) <= 255)), CONSTRAINT check_ef6176834f CHECK ((char_length(encrypted_cloud_license_auth_token_iv) <= 255)) ); diff --git a/doc/api/settings.md b/doc/api/settings.md index 0f0a36c0b5d..ada1d0e7fc4 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -277,7 +277,9 @@ listed in the descriptions of the relevant settings. | `elasticsearch_namespace_ids` | array of integers | no | **(PREMIUM)** The namespaces to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. | | `elasticsearch_project_ids` | array of integers | no | **(PREMIUM)** The projects to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. | | `elasticsearch_search` | boolean | no | **(PREMIUM)** Enable Elasticsearch search. | -| `elasticsearch_url` | string | no | **(PREMIUM)** The URL to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (for example, `http://:@:9200/`). | +| `elasticsearch_url` | string | no | **(PREMIUM)** The URL to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). | +| `elasticsearch_username` | string | no | **(PREMIUM)** The `username` of your Elasticsearch instance. | +| `elasticsearch_password` | string | no | **(PREMIUM)** The password of your Elasticsearch instance. | | `email_additional_text` | string | no | **(PREMIUM)** Additional text added to the bottom of every email for legal/auditing/compliance reasons. | | `email_author_in_body` | boolean | no | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. | | `enabled_git_access_protocol` | string | no | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. | diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 001dfc02d0b..01ac7de81de 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -229,7 +229,9 @@ The following Elasticsearch settings are available: | `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | | `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. | | `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | -| `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., `http://host1, https://host2:9200`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://:@:9200/`). Special characters in the username or password should use [percentage encoding](https://en.wikipedia.org/wiki/Percent-encoding). | +| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://:@:9200/`. | +| `Username` | The `username` of your Elasticsearch instance. | +| `Password` | The password of your Elasticsearch instance. | | `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indexes with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | | `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value will greatly increase total disk space required by the index. | | `Limit namespaces and projects that can be indexed` | Enabling this will allow you to select namespaces and projects to index. All other namespaces and projects will use database search instead. Please note that if you enable this option but do not select any namespaces or projects, none will be indexed. [Read more below](#limiting-namespaces-and-projects). @@ -326,16 +328,57 @@ index alias to it which becomes the new `primary` index. At the end, we resume t > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. > - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. +> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. -Under **Admin Area > Settings > Advanced Search > Elasticsearch zero-downtime reindexing**, click on **Trigger cluster reindexing**. +To trigger the reindexing process: + +1. Sign in to your GitLab instance as an administrator. +1. Go to **Admin Area > Settings > Advanced Search > Elasticsearch zero-downtime reindexing**. +1. Select **Trigger cluster reindexing**. Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. -WARNING: -After the reindexing is completed, the original index will be scheduled to be deleted after 14 days. You can cancel this action by pressing the cancel button. +After this process is completed, the original index is scheduled to be deleted after +14 days. You can cancel this action by pressing the **Cancel** button on the same +page you triggered the reindexing process. While the reindexing is running, you will be able to follow its progress under that same section. +#### Elasticsearch zero-downtime reindexing + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. + +The following reindex settings are available in **Admin Area > Settings > Advanced Search > Elasticsearch zero-downtime reindexing**: + +- [Slice multiplier](#slice-multiplier) +- [Maximum running slices](#maximum-running-slices) + +##### Slice multiplier + +The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice). + +GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice) +to control the reindex efficiently and safely, which enables users to retry only +failed slices. + +The multiplier defaults to `2` and applies to the number of shards per index. +For example, if this value is `2` and your index has 20 shards, then the +reindex task is split into 40 slices. + +##### Maximum running slices + +The maximum running slices parameter defaults to `60` and corresponds to the +maximum number of slices allowed to run concurrently during Elasticsearch +reindexing. + +Setting this value too high can have adverse performance impacts as your cluster +may become heavily saturated with searches and writes. Setting this value too +low may lead the reindexing process to take a very long time to complete. + +The best value for this will depend on your cluster size, whether you're willing +to accept some degraded search performance during reindexing, and how important +it is for the reindex to finish quickly and unpause indexing. + ### Mark the most recent reindex job as failed and resume the indexing Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps: diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 93b32bd9bef..cce17913f71 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -460,11 +460,14 @@ configured to act as a remote proxy and add the `Gitlab-DAST-Permission` header. ### API scan -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10928) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10928) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> - A new DAST API scanning engine was introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.10. Using an API specification as a scan's target is a useful way to seed URLs for scanning an API. Vulnerability rules in an API scan are different than those in a normal website scan. +A new DAST API scanning engine is available in GitLab 13.12 and later. For more details, see [DAST API scanning engine](../dast_api). The new scanning engine supports REST, SOAP, GraphQL, and generic APIs using forms, XML, and JSON. Testing can be performed using OpenAPI, Postman Collections, and HTTP Archive (HAR) documents. + #### Specification format API scans support OpenAPI V2 and OpenAPI V3 specifications. You can define these specifications using `JSON` or `YAML`. diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_collection_edit_variable.png b/doc/user/application_security/dast_api/img/dast_api_postman_collection_edit_variable.png new file mode 100644 index 00000000000..3a2799ecdc1 Binary files /dev/null and b/doc/user/application_security/dast_api/img/dast_api_postman_collection_edit_variable.png differ diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_environment_edit_variable.png b/doc/user/application_security/dast_api/img/dast_api_postman_environment_edit_variable.png new file mode 100644 index 00000000000..656ba7652cd Binary files /dev/null and b/doc/user/application_security/dast_api/img/dast_api_postman_environment_edit_variable.png differ diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_request_edit.png b/doc/user/application_security/dast_api/img/dast_api_postman_request_edit.png new file mode 100644 index 00000000000..3750af8f455 Binary files /dev/null and b/doc/user/application_security/dast_api/img/dast_api_postman_request_edit.png differ diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md new file mode 100644 index 00000000000..8596b31f82b --- /dev/null +++ b/doc/user/application_security/dast_api/index.md @@ -0,0 +1,1088 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference, howto +--- + +# DAST API **(ULTIMATE)** + +You can add dynamic application security testing of web APIs to your [GitLab CI/CD](../../../ci/README.md) pipelines. +This helps you discover bugs and potential security issues that other QA processes may miss. + +We recommend that you use DAST API testing in addition to [GitLab Secure](../index.md)'s +other security scanners and your own test processes. If you're using [GitLab CI/CD](../../../ci/README.md), +you can run DAST API tests as part your CI/CD workflow. + +## Requirements + +- One of the following web API types: + - REST API + - SOAP + - GraphQL + - Form bodies, JSON, or XML +- One of the following assets to provide APIs to test: + - OpenAPI v2 or v3 API definition + - Postman Collection v2.0 or v2.1 + - HTTP Archive (HAR) of API requests to test + +## When DAST API scans run + +When using the `DAST-API.gitlab-ci.yml` template, the defined jobs use the `dast` stage by default. To enable your `.gitlab-ci.yml` file must include the `dast` stage in your `stages` definition. To ensure DAST API scans the latest code, your CI pipeline should deploy changes to a test environment in a stage before the `dast` stage: + +```yaml +stages: + - build + - test + - deploy + - dast +``` + +Note that if your pipeline is configured to deploy to the same web server on each run, running a +pipeline while another is still running could cause a race condition in which one pipeline +overwrites the code from another. The API to scan should be excluded from changes for the duration +of a DAST API scan. The only changes to the API should be from the DAST API scanner. Be aware that +any changes made to the API (for example, by users, scheduled tasks, database changes, code +changes, other pipelines, or other scanners) during a scan could cause inaccurate results. + +## Enable DAST API scanning + +There are three ways to perform scans. See the configuration section for the one you wish to use: + +- [OpenAPI v2 or v3 specification](#openapi-specification) +- [HTTP Archive (HAR)](#http-archive-har) +- [Postman Collection v2.0 or v2.1](#postman-collection) + +Examples of various configurations can be found here: + +- [Example OpenAPI v2 specification project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/openapi-example) +- [Example HTTP Archive (HAR) project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/har-example) +- [Example Postman Collection project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/postman-example) +- [Example GraphQL project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/graphql-example) + +WARNING: +GitLab 14.0 will require that you place DAST API configuration files (for example, +`gitlab-dast-api-config.yml`) in your repository's `.gitlab` directory instead of your +repository's root. You can continue using your existing configuration files as they are, but +starting in GitLab 14.0, GitLab will not check your repository's root for configuration files. + +### OpenAPI Specification + +The [OpenAPI Specification](https://www.openapis.org/) (formerly the Swagger Specification) is an +API description format for REST APIs. This section shows you how to configure DAST API by using +an OpenAPI specification to provide information about the target API to test. OpenAPI specifications +are provided as a file system resource or URL. + +DAST API uses an OpenAPI document to generate the request body. When a request body is required, +the body generation is limited to these body types: + +- `application/x-www-form-urlencoded` +- `multipart/form-data` +- `application/json` + +Follow these steps to configure DAST API in GitLab with an OpenAPI specification: + +1. To use DAST API, you must [include](../../../ci/yaml/README.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) + that's provided as part of your GitLab installation. Add the following to your + `.gitlab-ci.yml` file: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + ``` + +1. The [configuration file](#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file, + substituting `Quick` for the profile you choose: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + ``` + +1. Provide the location of the OpenAPI specification. You can provide the specification as a file + or URL. Specify the location by adding the `DAST_API_OPENAPI` variable: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + ``` + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run DAST API against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. DAST API + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + + Here's an example of using `DAST_API_TARGET_URL`: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + ``` + +This is a minimal configuration for DAST API. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +WARNING: +**NEVER** run DAST API testing against a production server. Not only can it perform *any* function that the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting data. Only run DAST API scanning against a test server. + +### HTTP Archive (HAR) + +The [HTTP Archive format (HAR)](http://www.softwareishard.com/blog/har-12-spec/) +is an archive file format for logging HTTP transactions. When used with the GitLab DAST API scanner, HAR must contain records of calling the web API to test. The DAST API scanner extracts all the requests and +uses them to perform testing. + +You can use various tools to generate HAR files: + +- [Insomnia Core](https://insomnia.rest/): API client +- [Chrome](https://www.google.com/chrome/): Browser +- [Firefox](https://www.mozilla.org/en-US/firefox/): Browser +- [Fiddler](https://www.telerik.com/fiddler): Web debugging proxy +- [GitLab HAR Recorder](https://gitlab.com/gitlab-org/security-products/har-recorder): Command line + +WARNING: +HAR files may contain sensitive information such as authentication tokens, API keys, and session +cookies. We recommend that you review the HAR file contents before adding them to a repository. + +Follow these steps to configure DAST API to use a HAR file that provides information about the +target API to test: + +1. To use DAST API, you must [include](../../../ci/yaml/README.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) + that's provided as part of your GitLab installation. To do so, add the following to your + `.gitlab-ci.yml` file: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + ``` + +1. The [configuration file](#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file, + substituting `Quick` for the profile you choose: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + ``` + +1. Provide the location of the HAR specification. You can provide the specification as a file + or URL. [URL support was introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285020) in GitLab 13.10 and later. Specify the location by adding the `DAST_API_HAR` variable: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_HAR: test-api-recording.har + ``` + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run DAST API against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. DAST API + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + + Here's an example of using `DAST_API_TARGET_URL`: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_HAR: test-api-recording.har + DAST_API_TARGET_URL: http://test-deployment/ + ``` + +This is a minimal configuration for DAST API. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +WARNING: +**NEVER** run DAST API testing against a production server. Not only can it perform *any* function that +the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting +data. Only run DAST API against a test server. + +### Postman Collection + +The [Postman API Client](https://www.postman.com/product/api-client/) is a popular tool that +developers and testers use to call various types of APIs. The API definitions +[can be exported as a Postman Collection file](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-postman-data) +for use with DAST API. When exporting, make sure to select a supported version of Postman +Collection: v2.0 or v2.1. + +When used with the GitLab DAST API scanner, Postman Collections must contain definitions of the web API to +test with valid data. The DAST API scanner extracts all the API definitions and uses them to perform +testing. + +WARNING: +Postman Collection files may contain sensitive information such as authentication tokens, API keys, +and session cookies. We recommend that you review the Postman Collection file contents before adding +them to a repository. + +Follow these steps to configure DAST API to use a Postman Collection file that provides +information about the target API to test: + +1. To use DAST API, you must [include](../../../ci/yaml/README.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) + that's provided as part of your GitLab installation. To do so, add the following to your + `.gitlab-ci.yml` file: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + ``` + +1. The [configuration file](#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file, + substituting `Quick` for the profile you choose: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + ``` + +1. Provide the location of the Postman Collection specification. You can provide the specification as a file or URL. Specify the location by adding the `DAST_API_POSTMAN_COLLECTION` variable: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection_serviceA.json + ``` + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run DAST API against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. DAST API + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + + Here's an example of using `DAST_API_TARGET_URL`: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection_serviceA.json + DAST_API_TARGET_URL: http://test-deployment/ + ``` + +This is a minimal configuration for DAST API. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +WARNING: +**NEVER** run DAST API testing against a production server. Not only can it perform *any* function that +the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting +data. Only run DAST API against a test server. + +#### Postman variables + +Postman allows the developer to define placeholders that can be used in different parts of the +requests. These placeholders are called variables, as explained in [Using variables](https://learning.postman.com/docs/sending-requests/variables/). +You can use variables to store and reuse values in your requests and scripts. For example, you can +edit the collection to add variables to the document: + +![Edit collection variable tab View](img/dast_api_postman_collection_edit_variable.png) + +You can then use the variables in sections such as URL, headers, and others: + +![Edit request using variables View](img/dast_api_postman_request_edit.png) + +Variables can be defined at different [scopes](https://learning.postman.com/docs/sending-requests/variables/#variable-scopes) +(for example, Global, Collection, Environment, Local, and Data). In this example, they're defined at +the Environment scope: + +![Edit environment variables View](img/dast_api_postman_environment_edit_variable.png) + +When you export a Postman collection, only Postman collection variables are exported into the +Postman file. For example, Postman does not export environment-scoped variables into the Postman +file. + +By default, the DAST API scanner uses the Postman file to resolve Postman variable values. If a JSON file +is set in a GitLab CI environment variable `DAST_API_POSTMAN_COLLECTION_VARIABLES`, then the JSON +file takes precedence to get Postman variable values. + +Although Postman can export environment variables into a JSON file, the format is not compatible +with the JSON expected by `DAST_API_POSTMAN_COLLECTION_VARIABLES`. + +Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection_serviceA.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: variable-collection-dictionary.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +The file `variable-collection-dictionary.json` is a JSON document. This JSON is an object with +key-value pairs for properties. The keys are the variables' names, and the values are the variables' +values. For example: + + ```json + { + "base_url": "http://127.0.0.1/", + "token": "Token 84816165151" + } + ``` + +### Authentication + +Authentication is handled by providing the authentication token as a header or cookie. You can +provide a script that performs an authentication flow or calculates the token. + +#### HTTP Basic Authentication + +[HTTP basic authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) +is an authentication method built in to the HTTP protocol and used in conjunction with +[transport layer security (TLS)](https://en.wikipedia.org/wiki/Transport_Layer_Security). +To use HTTP basic authentication, two CI/CD variables are added to your `.gitlab-ci.yml` file: + +- `DAST_API_HTTP_USERNAME`: The username for authentication. +- `DAST_API_HTTP_PASSWORD`: The password for authentication. + +For the password, we recommended that you [create a CI/CD variable](../../../ci/variables/README.md#custom-cicd-variables) +(for example, `TEST_API_PASSWORD`) set to the password. You can create CI/CD variables from the +GitLab projects page at **Settings > CI/CD**, in the **Variables** section. Use that variable +as the value for `DAST_API_HTTP_PASSWORD`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_HAR: test-api-recording.har + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_HTTP_USERNAME: testuser + DAST_API_HTTP_PASSWORD: $TEST_API_PASSWORD +``` + +#### Bearer Tokens + +Bearer tokens are used by several different authentication mechanisms, including OAuth2 and JSON Web +Tokens (JWT). Bearer tokens are transmitted using the `Authorization` HTTP header. To use bearer +tokens with DAST API, you need one of the following: + +- A token that doesn't expire +- A way to generate a token that lasts the length of testing +- A Python script that DAST API can call to generate the token + +##### Token doesn't expire + +If the bearer token doesn't expire, use the `DAST_API_OVERRIDES_ENV` variable to provide it. This +variable's content is a JSON snippet that provides headers and cookies to add to DAST API's +outgoing HTTP requests. + +Follow these steps to provide the bearer token with `DAST_API_OVERRIDES_ENV`: + +1. [Create a CI/CD variable](../../../ci/variables/README.md#custom-cicd-variables), + for example `TEST_API_BEARERAUTH`, with the value + `{"headers":{"Authorization":"Bearer dXNlcm5hbWU6cGFzc3dvcmQ="}}` (substitute your token). You + can create CI/CD variables from the GitLab projects page at **Settings > CI/CD**, in the + **Variables** section. + +1. In your `.gitlab-ci.yml` file, set `DAST_API_OVERRIDES_ENV` to the variable you just created: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: $TEST_API_BEARERAUTH + ``` + +1. To validate that authentication is working, run an DAST API test and review the job logs + and the test API's application logs. + +##### Token generated at test runtime + +If the bearer token must be generated and doesn't expire during testing, you can provide to DAST API a file containing the token. A prior stage and job, or part of the DAST API job, can +generate this file. + +DAST API expects to receive a JSON file with the following structure: + +```json +{ + "headers" : { + "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +This file can be generated by a prior stage and provided to DAST API through the +`DAST_API_OVERRIDES_FILE` CI/CD variable. + +Set `DAST_API_OVERRIDES_FILE` in your `.gitlab-ci.yml` file: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: output/dast-api-overrides.json +``` + +To validate that authentication is working, run an DAST API test and review the job logs and +the test API's application logs. + +##### Token has short expiration + +If the bearer token must be generated and expires prior to the scan's completion, you can provide a +program or script for the DAST API scanner to execute on a provided interval. The provided script runs in +an Alpine Linux container that has Python 3 and Bash installed. If the Python script requires +additional packages, it must detect this and install the packages at runtime. + +The script must create a JSON file containing the bearer token in a specific format: + +```json +{ + "headers" : { + "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +You must provide three CI/CD variables, each set for correct operation: + +- `DAST_API_OVERRIDES_FILE`: JSON file the provided command generates. +- `DAST_API_OVERRIDES_CMD`: Command that generates the JSON file. +- `DAST_API_OVERRIDES_INTERVAL`: Interval (in seconds) to run command. + +For example: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: output/dast-api-overrides.json + DAST_API_OVERRIDES_CMD: renew_token.py + DAST_API_OVERRIDES_INTERVAL: 300 +``` + +To validate that authentication is working, run an DAST API test and review the job logs and +the test API's application logs. + +### Configuration files + +To get you started quickly, GitLab provides the configuration file +[`gitlab-dast-api-config.yml`](https://gitlab.com/gitlab-org/security-products/analyzers/dast/-/blob/master/config/gitlab-dast-api-config.yml). +This file has several testing profiles that perform various numbers of tests. The run time of each +profile increases as the test numbers go up. To use a configuration file, add it to your +repository's root as `.gitlab/gitlab-dast-api-config.yml`. + +#### Profiles + +The following profiles are pre-defined in the default configuration file. Profiles +can be added, removed, and modified by creating a custom configuration. + +##### Quick + +- Application Information Check +- Cleartext Authentication Check +- FrameworkDebugModeCheck +- Html Injection Check +- Insecure Http Methods Check +- JSON Hijacking Check +- JSON Injection Check +- Sensitive Information Check +- Session Cookie Check +- SQL Injection Check +- Token Check +- Xml Injection Check + +##### Full + +- Application Information Check +- Cleartext AuthenticationCheck +- Cors Check +- Dns Rebinding Check +- Framework Debug Mode Check +- Html Injection Check +- Insecure Http Methods Check +- JSON Hijacking Check +- JSON Injection Check +- Open Redirect Check +- Sensitive File Check +- Sensitive Information Check +- Session Cookie Check +- SQL Injection Check +- Tls Configuration Check +- Token Check +- Xml Injection Check + +### Available CI/CD variables + +| CI/CD variable | Description | +|------------------------------------------------------|--------------------| +| `DAST_API_VERSION` | Specify DAST API container version. Defaults to `latest`. | +| `DAST_API_TARGET_URL` | Base URL of API testing target. | +|[`DAST_API_CONFIG`](#configuration-files) | DAST API configuration file. Defaults to `.gitlab-dast-api.yml`. | +|[`DAST_API_PROFILE`](#configuration-files) | Configuration profile to use during testing. Defaults to `Quick`. | +|[`DAST_API_OPENAPI`](#openapi-specification) | OpenAPI specification file or URL. | +|[`DAST_API_HAR`](#http-archive-har) | HTTP Archive (HAR) file. | +|[`DAST_API_POSTMAN_COLLECTION`](#postman-collection) | Postman Collection file. | +|[`DAST_API_POSTMAN_COLLECTION_VARIABLES`](#postman-variables) | Path to a JSON file to extract postman variable values. | +|[`DAST_API_OVERRIDES_FILE`](#overrides) | Path to a JSON file containing overrides. | +|[`DAST_API_OVERRIDES_ENV`](#overrides) | JSON string containing headers to override. | +|[`DAST_API_OVERRIDES_CMD`](#overrides) | Overrides command. | +|[`DAST_API_OVERRIDES_INTERVAL`](#overrides) | How often to run overrides command in seconds. Defaults to `0` (once). | +|[`DAST_API_HTTP_USERNAME`](#http-basic-authentication) | Username for HTTP authentication. | +|[`DAST_API_HTTP_PASSWORD`](#http-basic-authentication) | Password for HTTP authentication. | +|`DAST_API_SERVICE_START_TIMEOUT` | How long to wait for target API to become available in seconds. Default is 300 seconds. | +|`DAST_API_TIMEOUT` | How long to wait for API responses in seconds. Default is 30 seconds. | + +### Overrides + +DAST API provides a method to add or override specific items in your request, for example: + +- Headers +- Cookies +- Query string +- Form data +- JSON nodes +- XML nodes + +You can use this to inject semantic version headers, authentication, and so on. The +[authentication section](#authentication) includes examples of using overrides for that purpose. + +Overrides use a JSON document, where each type of override is represented by a JSON object: + +```json +{ + "headers": { + "header1": "value", + "header2": "value" + }, + "cookies": { + "cookie1": "value", + "cookie2": "value" + }, + "query": { + "query-string1": "value", + "query-string2": "value" + }, + "body-form": { + "form-param1": "value", + "form-param1": "value", + }, + "body-json": { + "json-path1": "value", + "json-path2": "value", + }, + "body-xml" : { + "xpath1": "value", + "xpath2": "value", + } +} +``` + +Example of setting a single header: + +```json +{ + "headers": { + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +Example of setting both a header and cookie: + +```json +{ + "headers": { + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + }, + "cookies": { + "flags": "677" + } +} +``` + +Example usage for setting a `body-form` override: + +```json +{ + "body-form": { + "username": "john.doe" + } +} +``` + +The override engine uses `body-form` when the request body has only form-data content. + +Example usage for setting a `body-json` override: + +```json +{ + "body-json": { + "$.credentials.access-token": "iddqd!42.$" + } +} +``` + +Note that each JSON property name in the object `body-json` is set to a [JSON Path](https://goessner.net/articles/JsonPath/) +expression. The JSON Path expression `$.credentials.access-token` identifies the node to be +overridden with the value `iddqd!42.$`. The override engine uses `body-json` when the request body +has only [JSON](https://www.json.org/json-en.html) content. + +For example, if the body is set to the following JSON: + +```json +{ + "credentials" : { + "username" :"john.doe", + "access-token" : "non-valid-password" + } +} +``` + +It is changed to: + +```json +{ + "credentials" : { + "username" :"john.doe", + "access-token" : "iddqd!42.$" + } +} +``` + +Here's an example for setting a `body-xml` override. The first entry overrides an XML attribute and +the second entry overrides an XML element: + +```json +{ + "body-xml" : { + "/credentials/@isEnabled": "true", + "/credentials/access-token/text()" : "iddqd!42.$" + } +} +``` + +Note that each JSON property name in the object `body-xml` is set to an +[XPath v2](https://www.w3.org/TR/xpath20/) +expression. The XPath expression `/credentials/@isEnabled` identifies the attribute node to override +with the value `true`. The XPath expression `/credentials/access-token/text()` identifies the +element node to override with the value `iddqd!42.$`. The override engine uses `body-xml` when the +request body has only [XML](https://www.w3.org/XML/) +content. + +For example, if the body is set to the following XML: + +```xml + + john.doe + non-valid-password + +``` + +It is changed to: + +```xml + + john.doe + iddqd!42.$ + +``` + +You can provide this JSON document as a file or environment variable. You may also provide a command +to generate the JSON document. The command can run at intervals to support values that expire. + +#### Using a file + +To provide the overrides JSON as a file, the `DAST_API_OVERRIDES_FILE` CI/CD variable is set. The path is relative to the job current working directory. + +Here's an example `.gitlab-ci.yml`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: output/dast-api-overrides.json +``` + +#### Using a CI/CD variable + +To provide the overrides JSON as a CI/CD variable, use the `DAST_API_OVERRIDES_ENV` variable. +This allows you to place the JSON as variables that can be masked and protected. + +In this example `.gitlab-ci.yml`, the `DAST_API_OVERRIDES_ENV` variable is set directly to the JSON: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: '{"headers":{"X-API-Version":"2"}}' +``` + +In this example `.gitlab-ci.yml`, the `SECRET_OVERRIDES` variable provides the JSON. This is a +[group or instance level CI/CD variable defined in the UI](../../../ci/variables/README.md#instance-cicd-variables): + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: $SECRET_OVERRIDES +``` + +#### Using a command + +If the value must be generated or regenerated on expiration, you can provide a program or script for +the DAST API scanner to execute on a specified interval. The provided script runs in an Alpine Linux +container that has Python 3 and Bash installed. If the Python script requires additional packages, +it must detect this and install the packages at runtime. The script creates the overrides JSON file +as defined above. + +You must provide three CI/CD variables, each set for correct operation: + +- `DAST_API_OVERRIDES_FILE`: File generated by the provided command. +- `DAST_API_OVERRIDES_CMD`: Command to generate JSON file. +- `DAST_API_OVERRIDES_INTERVAL`: Interval in seconds to run command. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: output/dast-api-overrides.json + DAST_API_OVERRIDES_CMD: renew_token.py + DAST_API_OVERRIDES_INTERVAL: 300 +``` + +## Running your first scan + +When configured correctly, a CI/CD pipeline contains a `dast` stage and an `dast_api` job. The job only fails when an invalid configuration is provided. During normal operation, the job always succeeds even if vulnerabilities are identified during testing. + +Vulnerabilities are displayed on the **Security** pipeline tab with the suite name. When testing against the repositories default branch, the DAST API vulnerabilities are also shown on the Security & Compliance's Vulnerability Report page. + +To prevent an excessive number of reported vulnerabilities, the DAST API scanner limits the number of vulnerabilities it reports per operation. + +## Viewing DAST API vulnerabilities + +The DAST API analyzer produces a JSON report that is collected and used +[to populate the vulnerabilities into GitLab vulnerability screens](#view-details-of-a-dast-api-vulnerability). + +See [handling false positives](#handling-false-positives) for information about configuration changes you can make to limit the number of false positives reported. + +### View details of a DAST API vulnerability + +Follow these steps to view details of a vulnerability: + +1. You can view vulnerabilities in a project, or a merge request: + + - In a project, go to the project's **{shield}** **Security & Compliance > Vulnerability Report** + page. This page shows all vulnerabilities from the default branch only. + - In a merge request, go the merge request's **Security** section and click the **Expand** + button. DAST API vulnerabilities are available in a section labeled + **DAST detected N potential vulnerabilities**. Click the title to display the vulnerability + details. + +1. Click the vulnerabilities title to display the details. The table below describes these details. + + | Field | Description | + |:--------------------|:----------------------------------------------------------------------------------------| + | Description | Description of the vulnerability including what was modified. | + | Project | Namespace and project in which the vulnerability was detected. | + | Method | HTTP method used to detect the vulnerability. | + | URL | URL at which the vulnerability was detected. | + | Request | The HTTP request that caused the vulnerability. | + | Unmodified Response | Response from an unmodified request. This is what a normal working response looks like. | + | Actual Response | Response received from test request. | + | Evidence | How we determined a vulnerability occurred. | + | Identifiers | The DAST API check used to find this vulnerability. | + | Severity | Severity of the vulnerability. | + | Scanner Type | Scanner used to perform testing. | + +### Security Dashboard + +The Security Dashboard is a good place to get an overview of all the security vulnerabilities in your groups, projects and +pipelines. For more information, see the [Security Dashboard documentation](../security_dashboard/index.md). + +### Interacting with the vulnerabilities + +Once a vulnerability is found, you can interact with it. Read more on how to +[address the vulnerabilities](../vulnerabilities/index.md). + +## Handling False Positives + +False positives can be handled in several ways: + +- Dismiss the vulnerability. +- Some checks have several methods of detecting when a vulnerability is identified, called _Assertions_. + Assertions can also be turned off and configured. For example, the DAST API scanner by default uses HTTP + status codes to help identify when something is a real issue. If an API returns a 500 error during + testing, this creates a vulnerability. This isn't always desired, as some frameworks return 500 errors often. +- Turn off the Check producing the false positive. This prevents the check from generating any + vulnerabilities. Example checks are the SQL Injection Check, and JSON Hijacking Check. + +### Turn off a Check + +Checks perform testing of a specific type and can be turned on and off for specific configuration +profiles. The provided [configuration files](#configuration-files) define several profiles that you +can use. The profile definition in the configuration file lists all the checks that are active +during a scan. To turn off a specific check, remove it from the profile definition in the +configuration file. The profiles are defined in the `Profiles` section of the configuration file. + +Example profile definition: + +```yaml +Profiles: + - Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonHijackingCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + - Name: TokenCheck + - Name: XmlInjectionCheck +``` + +To turn off the JSON Hijacking Check you can remove these lines: + +```yaml + - Name: JsonHijackingCheck +``` + +This results in the following YAML: + +```yaml +- Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + - Name: TokenCheck + - Name: XmlInjectionCheck +``` + +### Turn off an Assertion for a Check + +Assertions detect vulnerabilities in tests produced by checks. Many checks support multiple Assertions such as Log Analysis, Response Analysis, and Status Code. When a vulnerability is found, the Assertion used is provided. To identify which Assertions are on by default, see the Checks default configuration in the configuration file. The section is called `Checks`. + +This example shows the SQL Injection Check: + +```yaml +- Name: SqlInjectionCheck + Configuration: + UserInjections: [] + Assertions: + - Name: LogAnalysisAssertion + - Name: ResponseAnalysisAssertion + - Name: StatusCodeAssertion +``` + +Here you can see three Assertions are on by default. A common source of false positives is +`StatusCodeAssertion`. To turn it off, modify its configuration in the `Profiles` section. This +example provides only the other two Assertions (`LogAnalysisAssertion`, +`ResponseAnalysisAssertion`). This prevents `SqlInjectionCheck` from using `StatusCodeAssertion`: + +```yaml +Profiles: + - Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonHijackingCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + Assertions: + - Name: LogAnalysisAssertion + - Name: ResponseAnalysisAssertion + - Name: TokenCheck + - Name: XmlInjectionCheck +``` + +## Troubleshooting + +### Failed to start scanner session (version header not found) + +The DAST API engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause of this issue is changing the `DAST_API_API` variable from its default. + +**Error message** + +- In [GitLab 13.11 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/323939), `Failed to start scanner session (version header not found).` +- In GitLab 13.10 and earlier, `API Security version header not found. Are you sure that you are connecting to the API Security server?`. + +**Solution** + +- Remove the `DAST_API_API` variable from the `.gitlab-ci.yml` file. The value will be inherited from the DAST API CI/CD template. We recommend this method instead of manually setting a value. +- If removing the variable is not possible, check to see if this value has changed in the latest version of the [DAST API CI/CD template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). If so, update the value in the `.gitlab-ci.yml` file. + +## Glossary + +- Assert: Assertions are detection modules used by checks to trigger a vulnerability. Many assertions have + configurations. A check can use multiple Assertions. For example, Log Analysis, Response Analysis, + and Status Code are common Assertions used together by checks. Checks with multiple Assertions + allow them to be turned on and off. +- Check: Performs a specific type of test, or performed a check for a type of vulnerability. For + example, the SQL Injection Check performs DAST testing for SQL Injection vulnerabilities. The DAST API scanner is comprised of several checks. Checks can be turned on and off in a profile. +- Profile: A configuration file has one or more testing profiles, or sub-configurations. You may + have a profile for feature branches and another with extra testing for a main branch. diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 4588a731f85..571f30bb904 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -37,6 +37,7 @@ GitLab uses the following tools to scan and report known vulnerabilities found i | [Dependency List](dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | | [Dependency Scanning](dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | | [Dynamic Application Security Testing (DAST)](dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | +| [DAST API](dast_api/index.md) **(ULTIMATE)** | Analyze running web APIs for known vulnerabilities. | | [API fuzzing](api_fuzzing/index.md) **(ULTIMATE)** | Find unknown bugs and vulnerabilities in web APIs with fuzzing. | | [Secret Detection](secret_detection/index.md) | Analyze Git history for leaked secrets. | | [Security Dashboard](security_dashboard/index.md) **(ULTIMATE)** | View vulnerabilities in all your projects and groups. | @@ -104,19 +105,6 @@ rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" ``` -## Security Scanning with Auto DevOps - -When [Auto DevOps](../../topics/autodevops/) is enabled, all GitLab Security scanning tools are configured using default settings. - -- [Auto SAST](../../topics/autodevops/stages.md#auto-sast) -- [Auto Secret Detection](../../topics/autodevops/stages.md#auto-secret-detection) -- [Auto DAST](../../topics/autodevops/stages.md#auto-dast) -- [Auto Dependency Scanning](../../topics/autodevops/stages.md#auto-dependency-scanning) -- [Auto License Compliance](../../topics/autodevops/stages.md#auto-license-compliance) -- [Auto Container Scanning](../../topics/autodevops/stages.md#auto-container-scanning) - -While you cannot directly customize Auto DevOps, you can [include the Auto DevOps template in your project's `.gitlab-ci.yml` file](../../topics/autodevops/customize.md#customizing-gitlab-ciyml). - ## Default behavior of GitLab security scanning tools ### Secure jobs in your pipeline diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb index b9fa1e0549c..e343e2dcf91 100644 --- a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'rails/generators' -require_relative '../usage_metric_definition_generator' module Gitlab module UsageMetricDefinition diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 1ba5f4fa4ce..59249c8bc1f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -243,23 +243,13 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil, port = nil) - env = Rails.env - original_config = ActiveRecord::Base.configurations.to_h + original_config = Gitlab::Database.config - env_config = original_config[env].with_indifferent_access.merge(pool: pool_size) + env_config = original_config.merge(pool: pool_size) env_config[:host] = host if host env_config[:port] = port if port - config = ActiveRecord::DatabaseConfigurations.new( - original_config.merge(env => env_config) - ) - - spec = - ActiveRecord:: - ConnectionAdapters:: - ConnectionSpecification::Resolver.new(config).spec(env.to_sym) - - ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(env_config) end def self.connection diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2afaf90e740..f52942882a1 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2777,9 +2777,6 @@ msgstr "" msgid "Advanced Search" msgstr "" -msgid "Advanced Search with Elasticsearch" -msgstr "" - msgid "Advanced Settings" msgstr "" @@ -8425,6 +8422,9 @@ msgstr "" msgid "Configure repository mirroring." msgstr "" +msgid "Configure settings for Advanced Search with Elasticsearch." +msgstr "" + msgid "Configure specific limits for Packages API requests that supersede the general user and IP rate limits." msgstr "" @@ -11945,6 +11945,9 @@ msgstr "" msgid "Elasticsearch reindexing triggered" msgstr "" +msgid "Elasticsearch reindexing was not started: %{errors}" +msgstr "" + msgid "Elasticsearch returned status code: %{status_code}" msgstr "" @@ -20266,6 +20269,9 @@ msgstr "" msgid "Maximum push size (MB)" msgstr "" +msgid "Maximum running slices" +msgstr "" + msgid "Maximum size limit for a single commit." msgstr "" @@ -23650,6 +23656,9 @@ msgstr "" msgid "Password" msgstr "" +msgid "Password (for password-protected Elasticsearch servers)" +msgstr "" + msgid "Password (optional)" msgstr "" @@ -26985,7 +26994,7 @@ msgstr "" msgid "Regulate approvals by authors/committers. Affects all projects." msgstr "" -msgid "Reindexing Status: %{status}" +msgid "Reindexing Status: %{status} (Slice multiplier: %{multiplier}, Maximum running slices: %{max_slices})" msgstr "" msgid "Rejected (closed)" @@ -30110,6 +30119,9 @@ msgstr "" msgid "SlackService|This service allows users to perform common operations on this project by entering slash commands in Slack." msgstr "" +msgid "Slice multiplier" +msgstr "" + msgid "Smartcard" msgstr "" @@ -32282,6 +32294,9 @@ msgstr "" msgid "The maximum file size is %{size}." msgstr "" +msgid "The maximum number of slices allowed to run concurrently during Elasticsearch reindexing. Learn more about %{max_slices_running_link_start}maximum running slices configuration%{max_slices_link_end}." +msgstr "" + msgid "The maximum number of tags that a single worker accepts for cleanup. If the number of tags goes above this limit, the list of tags to delete is truncated to this number. To remove this limit, set it to 0." msgstr "" @@ -33041,9 +33056,6 @@ msgstr "" msgid "This feature requires local storage to be enabled" msgstr "" -msgid "This feature should be used with an index that was created after 13.0" -msgstr "" - msgid "This field is required." msgstr "" @@ -34241,6 +34253,9 @@ msgstr "" msgid "Trigger cluster reindexing" msgstr "" +msgid "Trigger cluster reindexing. This feature should be used with an index that was created after 13.0." +msgstr "" + msgid "Trigger manual job" msgstr "" @@ -35201,6 +35216,9 @@ msgstr "" msgid "Used programming language" msgstr "" +msgid "Used to calculate the number of slices during reindexing. The multiplier will be applied to the number of shards per index. Learn more about %{slice_multiplier_link_start}slice multiplier configuration%{slice_multiplier_link_end}." +msgstr "" + msgid "Used to help configure your identity provider" msgstr "" @@ -35462,6 +35480,9 @@ msgstr "" msgid "UserProfile|made a private contribution" msgstr "" +msgid "Username (for password-protected Elasticsearch servers)" +msgstr "" + msgid "Username (optional)" msgstr ""