From 66fc7ba6f3a4a8e74529192f1dd110d87a91fdfb Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 15 Feb 2022 06:17:51 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab-ci.yml | 3 +- .gitlab/ci/frontend.gitlab-ci.yml | 2 +- .gitlab/ci/global.gitlab-ci.yml | 30 +++--- .gitlab/ci/workhorse.gitlab-ci.yml | 4 +- .../stylesheets/pages/merge_requests.scss | 4 + app/helpers/webpack_helper.rb | 12 ++- config/initializers/static_files.rb | 2 +- ...gitaly-deprecate-legacy-config-options.yml | 20 ++++ data/deprecations/15-0-oauth-noexpiry.yml | 24 +++++ doc/integration/oauth_provider.md | 21 ++--- doc/update/deprecations.md | 36 +++++++ .../dependency_scanning/index.md | 1 - doc/user/application_security/sast/index.md | 48 +++++++++- .../secret_detection/index.md | 75 +++++++++++++++ lib/tasks/gitlab/db.rake | 33 ++++--- .../add_design_content_spec.rb | 2 +- spec/models/user_spec.rb | 2 + spec/services/ci/register_job_service_spec.rb | 4 + spec/tasks/gitlab/db_rake_spec.rb | 93 +++++++++++++++++++ 19 files changed, 367 insertions(+), 49 deletions(-) create mode 100644 data/deprecations/14-8-gitaly-deprecate-legacy-config-options.yml create mode 100644 data/deprecations/15-0-oauth-noexpiry.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ca10a0c567..1b551967cb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ stages: # in cases where jobs require Docker-in-Docker, the job # definition must be extended with `.use-docker-in-docker` default: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 tags: - gitlab-org # All jobs are interruptible by default @@ -67,6 +67,7 @@ variables: GIT_DEPTH: "20" GIT_SUBMODULE_STRATEGY: "none" GET_SOURCES_ATTEMPTS: "3" + DEBIAN_VERSION: "bullseye" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 1bfcb8b4909..ef5af342544 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -11,7 +11,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 variables: SETUP_DB: "false" WEBPACK_VENDOR_DLL: "true" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 26f39af05e6..33b04faf731 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -18,7 +18,7 @@ - source scripts/prepare_build.sh .ruby-gems-cache: &ruby-gems-cache - key: "ruby-gems-v2" + key: "ruby-gems-${DEBIAN_VERSION}" paths: - vendor/ruby/ policy: pull @@ -28,7 +28,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .gitaly-ruby-gems-cache: &gitaly-ruby-gems-cache - key: "gitaly-ruby-gems-v2" + key: "gitaly-ruby-gems-${DEBIAN_VERSION}" paths: - vendor/gitaly-ruby/ policy: pull @@ -41,7 +41,7 @@ key: files: - GITALY_SERVER_VERSION - prefix: "gitaly-binaries" + prefix: "gitaly-binaries-${DEBIAN-VERSION}" paths: - tmp/tests/gitaly/_build/bin/ - tmp/tests/gitaly/_build/deps/git/install/ @@ -56,7 +56,7 @@ policy: pull .go-pkg-cache: &go-pkg-cache - key: "go-pkg-v1" + key: "go-pkg-${DEBIAN_VERSION}" paths: - .go/pkg/mod/ policy: pull @@ -66,7 +66,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .node-modules-cache: &node-modules-cache - key: "node-modules-${NODE_ENV}-v1" + key: "node-modules-${DEBIAN_VERSION}-${NODE_ENV}" paths: - node_modules/ - tmp/cache/webpack-dlls/ @@ -77,7 +77,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .assets-cache: &assets-cache - key: "assets-${NODE_ENV}-v2" + key: "assets-${DEBIAN_VERSION}-${NODE_ENV}" paths: - assets-hash.txt - public/assets/webpack/ @@ -91,7 +91,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .storybook-node-modules-cache: &storybook-node-modules-cache - key: "storybook-node-modules-${NODE_ENV}-v1" + key: "storybook-node-modules-${DEBIAN_VERSION}-${NODE_ENV}" paths: - storybook/node_modules/ policy: pull @@ -101,7 +101,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .rubocop-cache: &rubocop-cache - key: "rubocop-v1" + key: "rubocop-${DEBIAN_VERSION}" paths: - tmp/rubocop_cache/ policy: pull @@ -113,7 +113,7 @@ policy: push .qa-ruby-gems-cache: &qa-ruby-gems-cache - key: "qa-ruby-gems-v1" + key: "qa-ruby-gems-${DEBIAN_VERSION}" paths: - qa/vendor/ruby/ policy: pull @@ -213,7 +213,7 @@ - *storybook-node-modules-cache-push .use-pg11: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -222,7 +222,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -231,7 +231,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg13: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -240,7 +240,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg11-ee: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -251,7 +251,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12-ee: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -262,7 +262,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg13-ee: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-97-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index 11f8886e3f4..01e059b8a60 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -22,8 +22,8 @@ workhorse:verify: workhorse:test using go 1.16: extends: .workhorse:test - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7-golang-1.16-git-2.31 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-golang-1.16-git-2.31 workhorse:test using go 1.17: extends: .workhorse:test - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-bullseye-ruby-2.7-golang-1.17-git-2.31 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-golang-1.17-git-2.31 diff --git a/app/assets/stylesheets/pages/merge_requests.scss b/app/assets/stylesheets/pages/merge_requests.scss index dc01aea7681..f95cff012d0 100644 --- a/app/assets/stylesheets/pages/merge_requests.scss +++ b/app/assets/stylesheets/pages/merge_requests.scss @@ -218,6 +218,10 @@ $tabs-holder-z-index: 250; .epic-tabs-holder { top: $header-height; z-index: $tabs-holder-z-index; + margin-left: -$gl-padding; + margin-right: -$gl-padding; + padding-left: $gl-padding; + padding-right: $gl-padding; background-color: $body-bg; border-bottom: 1px solid $border-color; diff --git a/app/helpers/webpack_helper.rb b/app/helpers/webpack_helper.rb index ba3c232bec4..64900714327 100644 --- a/app/helpers/webpack_helper.rb +++ b/app/helpers/webpack_helper.rb @@ -83,8 +83,16 @@ module WebpackHelper end def webpack_public_host - # We proxy webpack output in 'test' and 'dev' environment, so we can just use asset_host - ActionController::Base.asset_host.try(:chomp, '/') + # We do not proxy the webpack output in the 'test' environment, + # so we must reference the webpack dev server directly. + if Rails.env.test? && Gitlab.config.webpack.dev_server.enabled + host = Gitlab.config.webpack.dev_server.host + port = Gitlab.config.webpack.dev_server.port + protocol = Gitlab.config.webpack.dev_server.https ? 'https' : 'http' + "#{protocol}://#{host}:#{port}" + else + ActionController::Base.asset_host.try(:chomp, '/') + end end def webpack_public_path diff --git a/config/initializers/static_files.rb b/config/initializers/static_files.rb index a26d78f102b..2879d48387d 100644 --- a/config/initializers/static_files.rb +++ b/config/initializers/static_files.rb @@ -21,7 +21,7 @@ if app.config.public_file_server.enabled # If webpack-dev-server is configured, proxy webpack's public directory # instead of looking for static assets - if Gitlab.config.webpack.dev_server.enabled && Gitlab.dev_or_test_env? + if Gitlab.config.webpack.dev_server.enabled && Rails.env.development? app.config.middleware.insert_before( Gitlab::Middleware::Static, Gitlab::Webpack::DevServerMiddleware, diff --git a/data/deprecations/14-8-gitaly-deprecate-legacy-config-options.yml b/data/deprecations/14-8-gitaly-deprecate-legacy-config-options.yml new file mode 100644 index 00000000000..861bed1e976 --- /dev/null +++ b/data/deprecations/14-8-gitaly-deprecate-legacy-config-options.yml @@ -0,0 +1,20 @@ +- name: "Deprecate legacy Gitaly configuration methods" # The name of the feature to be deprecated + announcement_milestone: "14.8" # The milestone when this feature was first announced as deprecated. + announcement_date: "2022-02-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + removal_milestone: "15.0" # The milestone when this feature is planned to be removed + removal_date: "2022-05-22" # The date of the milestone release when this feature is planned to be removed. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + breaking_change: true # If this deprecation is a breaking change, set this value to true + reporter: mjwood # GitLab username of the person reporting the deprecation + body: | # Do not modify this line, instead modify the lines below. + Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). + These variables are being replaced with standard [`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). + + GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using + `config.toml`. +# The following items are not published on the docs page, but may be used in the future. + stage: "Create" # (optional - may be required in the future) String value of the stage that the feature was created in. e.g., Growth + tiers: # (optional - may be required in the future) An array of tiers that the feature is available in currently. e.g., [Free, Silver, Gold, Core, Premium, Ultimate] + issue_url: "https://gitlab.com/gitlab-org/gitlab/-/issues/352609" # (optional) This is a link to the deprecation issue in GitLab + documentation_url: "https://docs.gitlab.com/ee/administration/gitaly/reference.html" # (optional) This is a link to the current documentation page + image_url: # (optional) This is a link to a thumbnail image depicting the feature + video_url: # (optional) Use the youtube thumbnail URL with the structure of https://img.youtube.com/vi/UNIQUEID/hqdefault.jpg diff --git a/data/deprecations/15-0-oauth-noexpiry.yml b/data/deprecations/15-0-oauth-noexpiry.yml new file mode 100644 index 00000000000..2e1e4a35f7b --- /dev/null +++ b/data/deprecations/15-0-oauth-noexpiry.yml @@ -0,0 +1,24 @@ +- name: "OAuth tokens without expiration" # The name of the feature to be deprecated + announcement_milestone: "14.8" # The milestone when this feature was first announced as deprecated. + announcement_date: "2022-02-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + removal_milestone: "15.0" # The milestone when this feature is planned to be removed + removal_date: 2022-05-22 # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + breaking_change: true # If this deprecation is a breaking change, set this value to true + body: | # Do not modify this line, instead modify the lines below. + By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens + had no expiration. In GitLab 15.0, an expiry will be automatically generated for any existing token that does not + already have one. + + You should [opt in](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens) to expiring + tokens before GitLab 15.0 is released: + + 1. Edit the application. + 1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire. + +# The following items are not published on the docs page, but may be used in the future. + stage: # Manage + tiers: # (optional - may be required in the future) An array of tiers that the feature is available in currently. e.g., [Free, Silver, Gold, Core, Premium, Ultimate] + issue_url: # https://gitlab.com/gitlab-org/gitlab/-/issues/21745 + documentation_url: # (optional) This is a link to the current documentation page + image_url: # (optional) This is a link to a thumbnail image depicting the feature + video_url: # (optional) Use the youtube thumbnail URL with the structure of https://img.youtube.com/vi/UNIQUEID/hqdefault.jpg diff --git a/doc/integration/oauth_provider.md b/doc/integration/oauth_provider.md index d091de09ee4..adfb2fad941 100644 --- a/doc/integration/oauth_provider.md +++ b/doc/integration/oauth_provider.md @@ -86,23 +86,20 @@ To create an application for your GitLab instance: When creating application in the **Admin Area** , you can mark it as _trusted_. The user authorization step is automatically skipped for this application. -## Expiring Access Tokens +## Expiring access tokens > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21745) in GitLab 14.3. -By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and -earlier, OAuth access tokens had no expiration. +WARNING: +The ability to opt-out of expiring access tokens [is deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/340848). +All existing integrations should be updated to support access token refresh. -All integrations should update to support access token refresh. +Access tokens expire in two hours which means that integrations that use them must support generating new access +tokens at least every two hours. Existing: -When creating new applications, you can opt-out of expiry for backward compatibility by clearing -**Expire access tokens** when creating them. The ability to opt-out -[is deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/340848). - -Existing: - -- Applications can have expiring access tokens. Edit the application and select - **Expire access tokens** to enable them. +- Applications can have expiring access tokens: + 1. Edit the application. + 1. Select **Expire access tokens**. - Tokens must be [revoked](../api/oauth2.md#revoke-a-token) or they don't expire. When applications are deleted, all grants and tokens associated with the application are also deleted. diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index f63cbab330d..70e3a7a6319 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -809,6 +809,22 @@ The following `geo:db:*` tasks will be replaced with their corresponding `db:*:g **Planned removal milestone: 15.0 (2022-05-22)** +### Deprecate legacy Gitaly configuration methods + +WARNING: +This feature will be changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). +These variables are being replaced with standard [`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). + +GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using +`config.toml`. + +**Planned removal milestone: 15.0 (2022-05-22)** + ### Elasticsearch 6.8 WARNING: @@ -851,6 +867,26 @@ To align with this change, API calls to list external status checks will also re **Planned removal milestone: 15.0 (2022-05-22)** +### OAuth tokens without expiration + +WARNING: +This feature will be changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens +had no expiration. In GitLab 15.0, an expiry will be automatically generated for any existing token that does not +already have one. + +You should [opt in](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens) to expiring +tokens before GitLab 15.0 is released: + +1. Edit the application. +1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire. + +**Planned removal milestone: 15.0 (2022-05-22)** + ### Optional enforcement of PAT expiration WARNING: diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index e6548dffd9a..5c4327717ab 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -569,7 +569,6 @@ The following variables are used for configuring specific analyzers (used for a | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | -| `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12296) in GitLab 12.1, [removed](https://www.python.org/doc/sunset-python-2/) in GitLab 13.7). | | `RETIREJS_JS_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json` | Path or URL to `retire.js` JS vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | | `RETIREJS_NODE_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/npmrepository.json` | Path or URL to `retire.js` node vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | | `RETIREJS_ADVISORY_DB_INSECURE` | `retire.js` | `false` | Enable fetching remote JS and Node vulnerability data files (defined by the `RETIREJS_JS_ADVISORY_DB` and `RETIREJS_NODE_ADVISORY_DB` variables) from hosts using an insecure or self-signed SSL (TLS) certificate. | diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 11e35a8ad73..304e9b752ba 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -288,12 +288,14 @@ brakeman-sast: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235382) in GitLab 13.5. > - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/339614) support for > passthrough chains. Expanded to include additional passthrough types of `file`, `git`, and `url` in GitLab 14.6. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/235359) support for overriding rules in GitLab 14.8. You can customize the default scanning rules provided by our SAST analyzers. -Ruleset customization supports two capabilities that can be used +Ruleset customization supports the following that can be used simultaneously: - [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). Available for all analyzers. +- [Overriding predefined rules](index.md#override-predefined-analyzer-rules). Available for all analyzers. - Modifying the default behavior of a given analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. To customize the default scanning rules, create a file containing custom rules. These rules @@ -343,6 +345,50 @@ and `sobelow` by matching the `type` and `value` of identifiers: value = "sql_injection" ``` +#### Override predefined analyzer rules + +To override analyzer rules: + +1. In one or more `ruleset.identifier` subsections, list the rules that you want to override. Every `ruleset.identifier` section has: + + - a `type` field, to name the predefined rule identifier that the targeted analyzer uses. + - a `value` field, to name the rule to be overridden. + +1. In the `ruleset.override` context of a `ruleset` section, + provide the keys to override. Any combination of keys can be + overridden. Valid keys are: + + - description + - message + - name + - severity (valid options are: Critical, High, Medium, Low, Unknown, Info) + +##### Example: Override predefined rules of SAST analyzers + +In the following example, rules from `eslint` +and `gosec` are matched by the `type` and `value` of identifiers and +then overridden: + +```toml +[eslint] + [[eslint.ruleset]] + [eslint.ruleset.identifier] + type = "eslint_rule_id" + value = "security/detect-object-injection" + [eslint.ruleset.override] + description = "OVERRIDDEN description" + message = "OVERRIDDEN message" + name = "OVERRIDDEN name" + severity = "Critical" +[gosec] + [[gosec.ruleset]] + [gosec.ruleset.identifier] + type = "CWE" + value = "CWE-79" + [gosec.ruleset.override] + severity = "Critical" +``` + #### Synthesize a custom configuration To create a custom configuration, you can use passthrough chains. diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 19d9cf18702..af41a164faf 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -182,14 +182,89 @@ Secret Detection can be customized by defining available CI/CD variables: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211387) in GitLab 13.5. > - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/339614) support for > passthrough chains. Expanded to include additional passthrough types of `file`, `git`, and `url` in GitLab 14.6. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/235359) support for overriding rules in GitLab 14.8. You can customize the default secret detection rules provided with GitLab. +Ruleset customization supports the following capabilities that can be used +simultaneously: + +- [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). +- [Overriding predefined rules](index.md#override-predefined-analyzer-rules). +- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. + Customization allows replacing the default secret detection rules with rules that you define. To create a custom ruleset: 1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. 1. Create a custom ruleset file named `secret-detection-ruleset.toml` in the `.gitlab` directory. + +#### Disable predefined analyzer rules + +To disable analyzer rules: + +1. Set the `disabled` flag to `true` in the context of a `ruleset` section. + +1. In one or more `ruleset.identifier` subsections, list the rules that you want disabled. Every `ruleset.identifier` section has: + + - a `type` field, to name the predefined rule identifier. + - a `value` field, to name the rule to be disabled. + +##### Example: Disable predefined rules of Secret Detection analyzer + +In the following example, the disabled rules is assigned to `secrets` +by matching the `type` and `value` of identifiers: + +```toml +[secrets] + [[secrets.ruleset]] + disable = true + [secrets.ruleset.identifier] + type = "gitleaks_rule_id" + value = "RSA private key" +``` + +#### Override predefined analyzer rules + +To override rules: + +1. In one or more `ruleset.identifier` subsections, list the rules that you want to override. Every `ruleset.identifier` section has: + + - a `type` field, to name the predefined rule identifier that the Secret Detection analyzer uses. + - a `value` field, to name the rule to be overridden. + +1. In the `ruleset.override` context of a `ruleset` section, + provide the keys to override. Any combination of keys can be + overridden. Valid keys are: + + - description + - message + - name + - severity (valid options are: Critical, High, Medium, Low, Unknown, Info) + +##### Example: Override predefined rules of Secret Detection analyzer + +In the following example, rules +are matched by the `type` and `value` of identifiers and +then overridden: + +```toml +[secrets] + [[secrets.ruleset]] + [secrets.ruleset.identifier] + type = "gitleaks_rule_id" + value = "RSA private key" + [secrets.ruleset.override] + description = "OVERRIDDEN description" + message = "OVERRIDDEN message" + name = "OVERRIDDEN name" + severity = "Info" +``` + +#### Synthesize a custom configuration + +To create a custom configuration, you can use passthrough chains. + 1. In the `secret-detection-ruleset.toml` file, do one of the following: - Define a custom ruleset: diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index efb0e1ef1e1..64277650d42 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -6,23 +6,32 @@ namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version' task :mark_migration_complete, [:version] => :environment do |_, args| - unless args[:version] - puts "Must specify a migration version as an argument".color(:red) + mark_migration_complete(args[:version]) + end + + namespace :mark_migration_complete do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + desc "Gitlab | DB | Manually insert schema migration version on #{name} database" + task name, [:version] => :environment do |_, args| + mark_migration_complete(args[:version], database: name) + end + end + end + + def mark_migration_complete(version, database: nil) + if version.to_i == 0 + puts 'Must give a version argument that is a non-zero integer'.color(:red) exit 1 end - version = args[:version].to_i - if version == 0 - puts "Version '#{args[:version]}' must be a non-zero integer".color(:red) - exit 1 - end + Gitlab::Database.database_base_models.each do |name, model| + next if database && database.to_s != name - sql = "INSERT INTO schema_migrations (version) VALUES (#{version})" - begin - ActiveRecord::Base.connection.execute(sql) - puts "Successfully marked '#{version}' as complete".color(:green) + model.connection.execute("INSERT INTO schema_migrations (version) VALUES (#{model.connection.quote(version)})") + + puts "Successfully marked '#{version}' as complete on database #{name}".color(:green) rescue ActiveRecord::RecordNotUnique - puts "Migration version '#{version}' is already marked complete".color(:yellow) + puts "Migration version '#{version}' is already marked complete on database #{name}".color(:yellow) end end diff --git a/qa/qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb b/qa/qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb index 5f896c7bf10..b7284f972ef 100644 --- a/qa/qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/design_management/add_design_content_spec.rb @@ -12,7 +12,7 @@ module QA Flow::Login.sign_in end - it 'user adds a design and annotates it', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347822' do + it 'user adds a design and annotates it', quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/352746', type: :investigating }, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347822' do issue.visit! Page::Project::Issue::Show.perform do |issue| diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 58f58232d52..e4f25c79e53 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -4194,6 +4194,8 @@ RSpec.describe User do context 'when FF ci_owned_runners_cross_joins_fix is disabled' do before do + skip_if_multiple_databases_are_setup + stub_feature_flags(ci_owned_runners_cross_joins_fix: false) end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 251159864f5..2127a4fa0fc 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -750,6 +750,8 @@ module Ci context 'with ci_queuing_use_denormalized_data_strategy disabled' do before do + skip_if_multiple_databases_are_setup + stub_feature_flags(ci_queuing_use_denormalized_data_strategy: false) end @@ -773,6 +775,8 @@ module Ci context 'when not using pending builds table' do before do + skip_if_multiple_databases_are_setup + stub_feature_flags(ci_pending_builds_queue_source: false) end diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 92c896b1ab0..4e9aac01c08 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -20,6 +20,99 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do allow(Rake::Task['db:seed_fu']).to receive(:invoke).and_return(true) end + describe 'mark_migration_complete' do + context 'with a single database' do + let(:main_model) { ActiveRecord::Base } + + before do + skip_if_multiple_databases_are_setup + end + + it 'marks the migration complete on the given database' do + expect(main_model.connection).to receive(:quote).and_call_original + expect(main_model.connection).to receive(:execute) + .with("INSERT INTO schema_migrations (version) VALUES ('123')") + + run_rake_task('gitlab:db:mark_migration_complete', '[123]') + end + end + + context 'with multiple databases' do + let(:main_model) { double(:model, connection: double(:connection)) } + let(:ci_model) { double(:model, connection: double(:connection)) } + let(:base_models) { { 'main' => main_model, 'ci' => ci_model } } + + before do + skip_if_multiple_databases_not_setup + + allow(Gitlab::Database).to receive(:database_base_models).and_return(base_models) + end + + it 'marks the migration complete on each database' do + expect(main_model.connection).to receive(:quote).with('123').and_return("'123'") + expect(main_model.connection).to receive(:execute) + .with("INSERT INTO schema_migrations (version) VALUES ('123')") + + expect(ci_model.connection).to receive(:quote).with('123').and_return("'123'") + expect(ci_model.connection).to receive(:execute) + .with("INSERT INTO schema_migrations (version) VALUES ('123')") + + run_rake_task('gitlab:db:mark_migration_complete', '[123]') + end + + context 'when the single database task is used' do + it 'marks the migration complete for the given database' do + expect(main_model.connection).to receive(:quote).with('123').and_return("'123'") + expect(main_model.connection).to receive(:execute) + .with("INSERT INTO schema_migrations (version) VALUES ('123')") + + expect(ci_model.connection).not_to receive(:quote) + expect(ci_model.connection).not_to receive(:execute) + + run_rake_task('gitlab:db:mark_migration_complete:main', '[123]') + end + end + end + + context 'when the migration is already marked complete' do + let(:main_model) { double(:model, connection: double(:connection)) } + let(:base_models) { { 'main' => main_model } } + + before do + allow(Gitlab::Database).to receive(:database_base_models).and_return(base_models) + end + + it 'prints a warning message' do + allow(main_model.connection).to receive(:quote).with('123').and_return("'123'") + + expect(main_model.connection).to receive(:execute) + .with("INSERT INTO schema_migrations (version) VALUES ('123')") + .and_raise(ActiveRecord::RecordNotUnique) + + expect { run_rake_task('gitlab:db:mark_migration_complete', '[123]') } + .to output(/Migration version '123' is already marked complete on database main/).to_stdout + end + end + + context 'when an invalid version is given' do + let(:main_model) { double(:model, connection: double(:connection)) } + let(:base_models) { { 'main' => main_model } } + + before do + allow(Gitlab::Database).to receive(:database_base_models).and_return(base_models) + end + + it 'prints an error and exits' do + expect(main_model).not_to receive(:quote) + expect(main_model.connection).not_to receive(:execute) + + expect { run_rake_task('gitlab:db:mark_migration_complete', '[abc]') } + .to output(/Must give a version argument that is a non-zero integer/).to_stdout + .and raise_error(SystemExit) { |error| expect(error.status).to eq(1) } + end + end + end + describe 'configure' do it 'invokes db:migrate when schema has already been loaded' do allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2])