From edcd73e5fdc11332fc3c974a76ad219928806bfd Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 10 Jun 2022 00:08:32 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/models/project.rb | 4 +++ .../security/configuration_presenter.rb | 3 +- app/views/layouts/terms.html.haml | 2 +- ..._for_vulnerability_reads_location_image.rb | 27 +++++++++++++++ db/schema_migrations/20220530044712 | 1 + db/structure.sql | 2 ++ doc/api/graphql/reference/index.md | 34 +++++++++++++++++++ doc/ci/variables/index.md | 2 +- doc/development/integrations/index.md | 2 +- doc/integration/elasticsearch.md | 13 +++++-- doc/topics/autodevops/quick_start_guide.md | 2 +- .../compliance/compliance_report/index.md | 2 +- doc/user/project/issues/design_management.md | 2 +- doc/user/project/web_ide/index.md | 2 +- .../DAST-On-Demand-API-Scan.gitlab-ci.yml | 12 +++++-- lib/tasks/gitlab/db/validate_config.rake | 4 ++- .../gitlab_migration_release_spec.rb | 16 +++++---- .../merge_when_pipeline_succeeds_spec.rb | 6 ++-- .../registrations/welcome_controller_spec.rb | 5 +++ spec/features/users/signup_spec.rb | 1 + spec/models/project_spec.rb | 8 +++++ .../gitlab/db/validate_config_rake_spec.rb | 5 ++- 22 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb create mode 100644 db/schema_migrations/20220530044712 diff --git a/app/models/project.rb b/app/models/project.rb index 88cc52429c1..aeed681fc6f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2916,6 +2916,10 @@ class Project < ApplicationRecord build_artifacts_size_refresh&.started? end + def security_training_available? + licensed_feature_available?(:security_training) + end + private # overridden in EE diff --git a/app/presenters/projects/security/configuration_presenter.rb b/app/presenters/projects/security/configuration_presenter.rb index 772be0125a0..8a6569e7bf3 100644 --- a/app/presenters/projects/security/configuration_presenter.rb +++ b/app/presenters/projects/security/configuration_presenter.rb @@ -24,7 +24,8 @@ module Projects gitlab_ci_history_path: gitlab_ci_history_path, auto_fix_enabled: autofix_enabled, can_toggle_auto_fix_settings: can_toggle_autofix, - auto_fix_user_path: auto_fix_user_path + auto_fix_user_path: auto_fix_user_path, + security_training_enabled: project.security_training_available? } end diff --git a/app/views/layouts/terms.html.haml b/app/views/layouts/terms.html.haml index 29f358778ea..c9baf0cd2b8 100644 --- a/app/views/layouts/terms.html.haml +++ b/app/views/layouts/terms.html.haml @@ -26,7 +26,7 @@ %li.header-user.dropdown = link_to current_user, class: user_dropdown_class, data: { toggle: "dropdown" } do = image_tag avatar_icon_for_user(current_user, 23), width: 23, height: 23, class: "header-user-avatar", data: { qa_selector: 'user_avatar' } - = sprite_icon('angle-down', css_class: 'caret-down') + = sprite_icon('chevron-down') .dropdown-menu.dropdown-menu-right = render 'layouts/header/current_user_dropdown' = yield diff --git a/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb b/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb new file mode 100644 index 00000000000..384924941c1 --- /dev/null +++ b/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddIndexForVulnerabilityReadsLocationImage < Gitlab::Database::Migration[2.0] + INDEX = :index_vulnerability_reads_on_location_image_partial + + disable_ddl_transaction! + + REPORT_TYPES = { + container_scanning: 2, + cluster_image_scanning: 7 + } + + QUERY = <<~SQL + report_type in (#{REPORT_TYPES[:container_scanning]}, #{REPORT_TYPES[:cluster_image_scanning]}) AND + location_image IS NOT NULL + SQL + + def up + add_concurrent_index :vulnerability_reads, [:project_id, :location_image], + where: QUERY, + name: INDEX + end + + def down + remove_concurrent_index_by_name :vulnerability_reads, INDEX + end +end diff --git a/db/schema_migrations/20220530044712 b/db/schema_migrations/20220530044712 new file mode 100644 index 00000000000..0277bc87f9f --- /dev/null +++ b/db/schema_migrations/20220530044712 @@ -0,0 +1 @@ +e76914c24f23d352e275387330a8731afb409fe9ae1168c1ee25f5acf5424c07 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 08c8c4c7be3..3f450b61b27 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -29807,6 +29807,8 @@ CREATE INDEX index_vulnerability_reads_on_cluster_agent_id ON vulnerability_read CREATE INDEX index_vulnerability_reads_on_location_image ON vulnerability_reads USING btree (location_image) WHERE (report_type = ANY (ARRAY[2, 7])); +CREATE INDEX index_vulnerability_reads_on_location_image_partial ON vulnerability_reads USING btree (project_id, location_image) WHERE ((report_type = ANY (ARRAY[2, 7])) AND (location_image IS NOT NULL)); + CREATE INDEX index_vulnerability_reads_on_scanner_id ON vulnerability_reads USING btree (scanner_id); CREATE UNIQUE INDEX index_vulnerability_reads_on_uuid ON vulnerability_reads USING btree (uuid); diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index ca79c040f0e..8b9e52945df 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -8752,6 +8752,29 @@ The connection type for [`Vulnerability`](#vulnerability). | `nodes` | [`[Vulnerability]`](#vulnerability) | A list of nodes. | | `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +#### `VulnerabilityContainerImageConnection` + +The connection type for [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[VulnerabilityContainerImageEdge]`](#vulnerabilitycontainerimageedge) | A list of edges. | +| `nodes` | [`[VulnerabilityContainerImage]`](#vulnerabilitycontainerimage) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `VulnerabilityContainerImageEdge` + +The edge type for [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage) | The item at the end of the edge. | + #### `VulnerabilityEdge` The edge type for [`Vulnerability`](#vulnerability). @@ -14754,6 +14777,7 @@ Represents vulnerability finding of a security report on the pipeline. | `topics` | [`[String!]`](#string) | List of project topics. | | `userPermissions` | [`ProjectPermissions!`](#projectpermissions) | Permissions for the current user on the resource. | | `visibility` | [`String`](#string) | Visibility of the project. | +| `vulnerabilityImages` | [`VulnerabilityContainerImageConnection`](#vulnerabilitycontainerimageconnection) | Container images reported on the project vulnerabilities. (see [Connections](#connections)) | | `vulnerabilityScanners` | [`VulnerabilityScannerConnection`](#vulnerabilityscannerconnection) | Vulnerability scanners reported on the project vulnerabilities. (see [Connections](#connections)) | | `webUrl` | [`String`](#string) | Web URL of the project. | | `wikiEnabled` | [`Boolean`](#boolean) | Indicates if Wikis are enabled for the current user. | @@ -17469,6 +17493,16 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | `linkType` | [`VulnerabilityIssueLinkType`](#vulnerabilityissuelinktype) | Filter issue links by link type. | +### `VulnerabilityContainerImage` + +Represents a container image reported on the related vulnerability. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `name` | [`String`](#string) | Name of the container image. | + ### `VulnerabilityDetailBase` Represents the vulnerability details base. diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index 53a695780c6..c53fad69376 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -373,7 +373,7 @@ You can configure a project, group, or instance CI/CD variable to be available only to pipelines that run on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). -[Merged results pipelines](../pipelines/merge_request_pipelines.md#types-of-merge-request-pipelines), which run on a +[Merged results pipelines](../pipelines/merged_results_pipelines.md), which run on a temporary merge commit, not a branch or tag, do not have access to these variables. Pipelines that run directly on the merge request's source branch, with no added merge commit, can access diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index e595fea6d96..604e481a809 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -43,7 +43,7 @@ if you need clarification or spot any outdated information. ### Define properties Integrations can define arbitrary properties to store their configuration with the class method `Integration.prop_accessor`. -The values are stored as a serialized JSON hash in the `integrations.properties` column. +The values are stored as an encrypted JSON hash in the `integrations.encrypted_properties` column. For example: diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 47afe7016d9..6574a13be9b 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -502,12 +502,19 @@ In addition to the Rake tasks, there are some environment variables that can be | `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | | `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | -### Indexing a specific project +### Indexing a range of projects or a specific project -Because the `ID_TO` and `ID_FROM` environment variables use the `or equal to` comparison, you can index only one project by using both these variables with the same project ID number: +Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing. ```shell -root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_TO=5 ID_FROM=5 +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100 +``` + +Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project +by setting both to the same project ID: + +```shell +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5 Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! ``` diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 02802ae89af..e883e8cda4d 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -236,7 +236,7 @@ you to common environment tasks: - **Monitoring** (**{chart}**) - Opens the metrics page where Prometheus collects data about the Kubernetes cluster and how the application affects it in terms of memory usage, CPU usage, and latency -- **Deploy to** (**{play}** **{chevron-down}**) - Displays a list of environments you can deploy to +- **Deploy to** (**{play}** **{chevron-lg-down}**) - Displays a list of environments you can deploy to - **Terminal** (**{terminal}**) - Opens a [web terminal](../../ci/environments/index.md#web-terminals-deprecated) session inside the container where the application is running - **Re-deploy to environment** (**{repeat}**) - For more information, see diff --git a/doc/user/compliance/compliance_report/index.md b/doc/user/compliance/compliance_report/index.md index 77dbefa0755..0006ae02752 100644 --- a/doc/user/compliance/compliance_report/index.md +++ b/doc/user/compliance/compliance_report/index.md @@ -113,7 +113,7 @@ You can generate a commit-specific Chain of Custody report for a given commit SH 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Compliance report**. -1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{angle-down}**). +1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{chevron-lg-down}**). 1. Enter the merge commit SHA, and then select **Export commit custody report**. SHA and then select **Export commit custody report**. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 02c311cf9a2..d1b27f6eab0 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -92,7 +92,7 @@ The design you selected opens. You can then [zoom in](#zoom-in-on-a-design) on i When viewing a design, you can move to other designs. To do so, either: -- In the top-right corner, select **Go to previous design** (**{chevron-left}**) or **Go to next design** (**{chevron-right}**). +- In the top-right corner, select **Go to previous design** (**{chevron-lg-left}**) or **Go to next design** (**{chevron-lg-right}**). - Press Left or Right on your keyboard. To return to the issue view, either: diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 731d09df089..facaba45aec 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -19,7 +19,7 @@ and from merge requests: - *When viewing a file, or the repository file list* - 1. In the upper right corner of the page, select **Open in Web IDE** if it is visible. 1. If **Open in Web IDE** is not visible: - 1. Select the (**{chevron-down}**) next to **Edit** or **Gitpod**, depending on your configuration. + 1. Select the (**{chevron-lg-down}**) next to **Edit** or **Gitpod**, depending on your configuration. 1. Select **Open in Web IDE** from the list to display it as the editing option. 1. Select **Open in Web IDE** to open the editor. - *When viewing a merge request* - diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 3f9c87b7abf..4a72f5e72b1 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml + stages: - build - test @@ -6,12 +11,13 @@ stages: variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - DAST_API_VERSION: "1" - DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + DAST_API_VERSION: "2" + DAST_API_IMAGE_SUFFIX: "" + DAST_API_IMAGE: api-security dast: stage: dast - image: $DAST_API_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true script: - /peach/analyzer-dast-api diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index 3debb1cd344..2a3a54b5351 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -141,7 +141,9 @@ namespace :gitlab do rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" rescue ActiveRecord::NoDatabaseError - rescue PG::ReadOnlySqlTransaction => err + rescue ActiveRecord::StatementInvalid => err + raise unless err.cause.is_a?(PG::ReadOnlySqlTransaction) + warn "WARNING: Could not write to the database #{db_config.name}: #{err.message}" end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_release_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_release_spec.rb index 201b8efdf6a..6910b6a7fa2 100644 --- a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_release_spec.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_release_spec.rb @@ -29,7 +29,7 @@ module QA # @param [Hash] release # @return [Hash] def comparable_release(release) - release&.except(:_links, :evidences)&.merge( + release&.except(:_links)&.merge( { author: release[:author].except(:web_url), commit: release[:commit].except(:web_url), @@ -42,12 +42,14 @@ module QA }), milestones: release[:milestones].map do |milestone| milestone.except(:id, :project_id).merge({ web_url: milestone[:web_url].split("/-/").last }) - end - # TODO: Add back evidence testing once implemented - # https://gitlab.com/gitlab-org/gitlab/-/issues/360567 - # evidences: release[:evidences].map do |evidence| - # evidence.merge({ filepath: evidence[:filepath].split("/-/").last }) - # end + end, + # evidences are not directly migrated but rather recreated on the same releases, + # so we only check the json file is there + evidences: release[:evidences].map do |evidence| + evidence + .except(:collected_at, :sha) + .merge({ filepath: evidence[:filepath].split("/-/").last.gsub(/\d+\.json/, "*.json") }) + end } ) end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb index 9c55a6cf9a3..ac53357a86f 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/merge_when_pipeline_succeeds_spec.rb @@ -32,7 +32,7 @@ module QA content: <<~YAML test: tags: ["#{runner_name}"] - script: sleep 5 + script: sleep 15 only: - merge_requests YAML @@ -62,8 +62,6 @@ module QA merge_request.description = Faker::Lorem.sentence merge_request.target_new_branch = false merge_request.source_branch = "mr-test-#{SecureRandom.hex(6)}-#{i + 1}" - merge_request.file_name = Faker::Lorem.word - merge_request.file_content = Faker::Lorem.sentence end # Load the page so that the browser is as prepared as possible to display the pipeline in progress when we @@ -91,7 +89,7 @@ module QA end aggregate_failures do - expect { mr.merged? }.to eventually_be_truthy.within(max_duration: 60), "Expected content 'The changes were merged' but it did not appear." + expect { mr.merged? }.to eventually_be_truthy.within(max_duration: 120), "Expected content 'The changes were merged' but it did not appear." expect(merge_request.reload!.merge_when_pipeline_succeeds).to be_truthy expect(merge_request.state).to eq('merged') expect(project.pipelines.last[:status]).to eq('success') diff --git a/spec/controllers/registrations/welcome_controller_spec.rb b/spec/controllers/registrations/welcome_controller_spec.rb index c444875bf74..8a5a8490a23 100644 --- a/spec/controllers/registrations/welcome_controller_spec.rb +++ b/spec/controllers/registrations/welcome_controller_spec.rb @@ -31,6 +31,7 @@ RSpec.describe Registrations::WelcomeController do context 'when role and setup_for_company is set' do before do + stub_feature_flags(about_your_company_registration_flow: false) user.update!(setup_for_company: false) sign_in(user) end @@ -60,6 +61,10 @@ RSpec.describe Registrations::WelcomeController do end describe '#update' do + before do + stub_feature_flags(about_your_company_registration_flow: false) + end + subject(:update) do patch :update, params: { user: { role: 'software_developer', setup_for_company: 'false' } } end diff --git a/spec/features/users/signup_spec.rb b/spec/features/users/signup_spec.rb index 3eae4955167..30441dac7b6 100644 --- a/spec/features/users/signup_spec.rb +++ b/spec/features/users/signup_spec.rb @@ -341,6 +341,7 @@ RSpec.describe 'Signup' do end it 'redirects to step 2 of the signup process, sets the role and redirects back' do + stub_feature_flags(about_your_company_registration_flow: false) visit new_user_registration_path fill_in_signup_form diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index bb65440d01e..794516fb544 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -8387,6 +8387,14 @@ RSpec.describe Project, factory_default: :keep do end end + describe '#security_training_available?' do + subject { build(:project) } + + it 'returns false' do + expect(subject.security_training_available?).to eq false + end + end + private def finish_job(export_job) diff --git a/spec/tasks/gitlab/db/validate_config_rake_spec.rb b/spec/tasks/gitlab/db/validate_config_rake_spec.rb index 6303215bbf4..03d7504e8b1 100644 --- a/spec/tasks/gitlab/db/validate_config_rake_spec.rb +++ b/spec/tasks/gitlab/db/validate_config_rake_spec.rb @@ -213,8 +213,11 @@ RSpec.describe 'gitlab:db:validate_config', :silence_stdout do } end + let(:exception) { ActiveRecord::StatementInvalid.new("READONLY") } + before do - expect(ActiveRecord::InternalMetadata).to receive(:upsert).at_least(:once).and_raise(PG::ReadOnlySqlTransaction, "READONLY") + allow(exception).to receive(:cause).and_return(PG::ReadOnlySqlTransaction.new("cannot execute INSERT in a read-only transaction")) + allow(ActiveRecord::InternalMetadata).to receive(:upsert).at_least(:once).and_raise(exception) end it_behaves_like 'validates successfully'