From 915ce2e09259d71275f3e0a6e5214b793c2eb52e Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 31 Aug 2022 21:12:55 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../dependency_proxy/group_settings/update.rb | 5 + .../development/s3_omit_multipart_urls.yml | 8 -- .../detect_cross_database_modification.yml | 3 +- .../query_analyzer_gitlab_schema_metrics.yml | 3 +- .../sbom_vulnerable_component_versions.yml | 11 +++ db/docs/vulnerability_advisories.yml | 11 +++ ...5152905_create_vulnerability_advisories.rb | 25 +++++ ...eate_sbom_vulnerable_component_versions.rb | 18 ++++ ...y_to_sbom_vulnerable_component_versions.rb | 19 ++++ ...y_to_sbom_vulnerable_component_versions.rb | 19 ++++ db/schema_migrations/20220815152905 | 1 + db/schema_migrations/20220816135816 | 1 + db/schema_migrations/20220819153725 | 1 + db/schema_migrations/20220819162852 | 1 + db/structure.sql | 75 ++++++++++++++ doc/api/graphql/reference/index.md | 2 + .../documentation/styleguide/word_list.md | 10 +- doc/development/integrations/secure.md | 24 ++--- .../testing_guide/best_practices.md | 2 +- .../terraform_module_registry/index.md | 2 +- lib/gitlab/database/gitlab_schemas.yml | 2 + .../query_analyzers/gitlab_schemas_metrics.rb | 2 +- .../prevent_cross_database_modification.rb | 2 +- lib/object_storage/direct_upload.rb | 2 +- spec/lib/object_storage/direct_upload_spec.rb | 98 ++++++++----------- ...sbom_vulnerable_component_versions_spec.rb | 23 +++++ ...sbom_vulnerable_component_versions_spec.rb | 23 +++++ 27 files changed, 304 insertions(+), 89 deletions(-) delete mode 100644 config/feature_flags/development/s3_omit_multipart_urls.yml rename config/feature_flags/{development => ops}/detect_cross_database_modification.yml (67%) rename config/feature_flags/{development => ops}/query_analyzer_gitlab_schema_metrics.yml (67%) create mode 100644 db/docs/sbom_vulnerable_component_versions.yml create mode 100644 db/docs/vulnerability_advisories.yml create mode 100644 db/migrate/20220815152905_create_vulnerability_advisories.rb create mode 100644 db/migrate/20220816135816_create_sbom_vulnerable_component_versions.rb create mode 100644 db/migrate/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions.rb create mode 100644 db/migrate/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions.rb create mode 100644 db/schema_migrations/20220815152905 create mode 100644 db/schema_migrations/20220816135816 create mode 100644 db/schema_migrations/20220819153725 create mode 100644 db/schema_migrations/20220819162852 create mode 100644 spec/migrations/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions_spec.rb create mode 100644 spec/migrations/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions_spec.rb diff --git a/app/graphql/mutations/dependency_proxy/group_settings/update.rb b/app/graphql/mutations/dependency_proxy/group_settings/update.rb index 65c919db3c3..6be07edd883 100644 --- a/app/graphql/mutations/dependency_proxy/group_settings/update.rb +++ b/app/graphql/mutations/dependency_proxy/group_settings/update.rb @@ -8,6 +8,11 @@ module Mutations include Mutations::ResolvesGroup + description 'These settings can be adjusted by the group Owner or Maintainer. However, in GitLab 16.0, we ' \ + 'will be limiting this to the Owner role. ' \ + '[GitLab-#364441](https://gitlab.com/gitlab-org/gitlab/-/issues/364441) proposes making ' \ + 'this change to match the permissions level in the user interface.' + authorize :admin_dependency_proxy argument :group_path, diff --git a/config/feature_flags/development/s3_omit_multipart_urls.yml b/config/feature_flags/development/s3_omit_multipart_urls.yml deleted file mode 100644 index 92d30601739..00000000000 --- a/config/feature_flags/development/s3_omit_multipart_urls.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: s3_omit_multipart_urls -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85306 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/359718 -milestone: '14.10' -type: development -group: group::package -default_enabled: true diff --git a/config/feature_flags/development/detect_cross_database_modification.yml b/config/feature_flags/ops/detect_cross_database_modification.yml similarity index 67% rename from config/feature_flags/development/detect_cross_database_modification.yml rename to config/feature_flags/ops/detect_cross_database_modification.yml index 7f74e136291..5f496118c4b 100644 --- a/config/feature_flags/development/detect_cross_database_modification.yml +++ b/config/feature_flags/ops/detect_cross_database_modification.yml @@ -1,8 +1,7 @@ --- name: detect_cross_database_modification introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73316 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344620 milestone: '14.5' -type: development +type: ops group: group::sharding default_enabled: false diff --git a/config/feature_flags/development/query_analyzer_gitlab_schema_metrics.yml b/config/feature_flags/ops/query_analyzer_gitlab_schema_metrics.yml similarity index 67% rename from config/feature_flags/development/query_analyzer_gitlab_schema_metrics.yml rename to config/feature_flags/ops/query_analyzer_gitlab_schema_metrics.yml index b784105368c..9793eebb014 100644 --- a/config/feature_flags/development/query_analyzer_gitlab_schema_metrics.yml +++ b/config/feature_flags/ops/query_analyzer_gitlab_schema_metrics.yml @@ -1,8 +1,7 @@ --- name: query_analyzer_gitlab_schema_metrics introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73839 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/345034 milestone: '14.5' -type: development +type: ops group: group::sharding default_enabled: false diff --git a/db/docs/sbom_vulnerable_component_versions.yml b/db/docs/sbom_vulnerable_component_versions.yml new file mode 100644 index 00000000000..147af9c046b --- /dev/null +++ b/db/docs/sbom_vulnerable_component_versions.yml @@ -0,0 +1,11 @@ +--- +table_name: sbom_vulnerable_component_versions +classes: +- Sbom::VulnerableComponentVersion +feature_categories: +- container_scanning +- dependency_scanning +- license_compliance +description: Stores information about vulnerable SBoM components +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95622 +milestone: '15.4' diff --git a/db/docs/vulnerability_advisories.yml b/db/docs/vulnerability_advisories.yml new file mode 100644 index 00000000000..2c88be94a11 --- /dev/null +++ b/db/docs/vulnerability_advisories.yml @@ -0,0 +1,11 @@ +--- +table_name: vulnerability_advisories +classes: +- Vulnerabilities::Advisory +feature_categories: +- container_scanning +- dependency_scanning +- license_compliance +description: Stores vulnerability advisories +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95622 +milestone: '15.4' diff --git a/db/migrate/20220815152905_create_vulnerability_advisories.rb b/db/migrate/20220815152905_create_vulnerability_advisories.rb new file mode 100644 index 00000000000..f01a17c6352 --- /dev/null +++ b/db/migrate/20220815152905_create_vulnerability_advisories.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class CreateVulnerabilityAdvisories < Gitlab::Database::Migration[2.0] + def change + create_table :vulnerability_advisories, id: false do |t| + t.uuid :uuid, null: false + t.timestamps_with_timezone null: false + t.primary_key :id + t.date :created_date, null: false + t.date :published_date, null: false + t.text :description, limit: 2048 + t.text :title, limit: 2048 + t.text :component_name, limit: 2048 + t.text :solution, limit: 2048 + t.text :not_impacted, limit: 2048 + t.text :cvss_v2, limit: 128 + t.text :cvss_v3, limit: 128 + t.text :affected_range, limit: 32 + t.text :identifiers, array: true, default: [] + t.text :fixed_versions, array: true, default: [] + t.text :urls, array: true, default: [] + t.text :links, array: true, default: [] + end + end +end diff --git a/db/migrate/20220816135816_create_sbom_vulnerable_component_versions.rb b/db/migrate/20220816135816_create_sbom_vulnerable_component_versions.rb new file mode 100644 index 00000000000..6f50376f6fa --- /dev/null +++ b/db/migrate/20220816135816_create_sbom_vulnerable_component_versions.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class CreateSbomVulnerableComponentVersions < Gitlab::Database::Migration[2.0] + ADVISORY_INDEX_NAME = "index_vulnerable_component_versions_on_vulnerability_advisory" + SBOM_COMPONENT_INDEX_NAME = "index_vulnerable_component_versions_on_sbom_component_version" + + def change + create_table :sbom_vulnerable_component_versions do |t| + t.references :vulnerability_advisory, + index: { name: ADVISORY_INDEX_NAME } + + t.references :sbom_component_version, + index: { name: SBOM_COMPONENT_INDEX_NAME } + + t.timestamps_with_timezone null: false + end + end +end diff --git a/db/migrate/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions.rb b/db/migrate/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions.rb new file mode 100644 index 00000000000..7f0c817875f --- /dev/null +++ b/db/migrate/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddVulnerabilityAdvisoryForeignKeyToSbomVulnerableComponentVersions < Gitlab::Database::Migration[2.0] + SOURCE_TABLE = :sbom_vulnerable_component_versions + TARGET_TABLE = :vulnerability_advisories + COLUMN = :vulnerability_advisory_id + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key SOURCE_TABLE, TARGET_TABLE, column: COLUMN, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key SOURCE_TABLE, column: COLUMN + end + end +end diff --git a/db/migrate/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions.rb b/db/migrate/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions.rb new file mode 100644 index 00000000000..a4f2005693c --- /dev/null +++ b/db/migrate/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddSbomComponentVersionForeignKeyToSbomVulnerableComponentVersions < Gitlab::Database::Migration[2.0] + SOURCE_TABLE = :sbom_vulnerable_component_versions + TARGET_TABLE = :sbom_component_versions + COLUMN = :sbom_component_version_id + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key SOURCE_TABLE, TARGET_TABLE, column: COLUMN, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key SOURCE_TABLE, column: COLUMN + end + end +end diff --git a/db/schema_migrations/20220815152905 b/db/schema_migrations/20220815152905 new file mode 100644 index 00000000000..9e71fe96edc --- /dev/null +++ b/db/schema_migrations/20220815152905 @@ -0,0 +1 @@ +8494a4a9c708ddfc63d86891b83f01c0883d8b88ebea2177980780a381d74704 \ No newline at end of file diff --git a/db/schema_migrations/20220816135816 b/db/schema_migrations/20220816135816 new file mode 100644 index 00000000000..ff2b64004dd --- /dev/null +++ b/db/schema_migrations/20220816135816 @@ -0,0 +1 @@ +47dcd45d2b9c35c4e3ca707d54a983e8321dd86b2b66d4bccca9001884bc6d13 \ No newline at end of file diff --git a/db/schema_migrations/20220819153725 b/db/schema_migrations/20220819153725 new file mode 100644 index 00000000000..617303214e4 --- /dev/null +++ b/db/schema_migrations/20220819153725 @@ -0,0 +1 @@ +5819791e71e9118680e24eceef92364c78ed51dda375db9902f693147ddd9765 \ No newline at end of file diff --git a/db/schema_migrations/20220819162852 b/db/schema_migrations/20220819162852 new file mode 100644 index 00000000000..b1d1fcb4c70 --- /dev/null +++ b/db/schema_migrations/20220819162852 @@ -0,0 +1 @@ +1e0109c4e1a0512864f3ed16d0a9bc82b40b5c6fb1586acaffe18191821df18f \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7a88015661a..a5f072d7c84 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -20856,6 +20856,23 @@ CREATE SEQUENCE sbom_sources_id_seq ALTER SEQUENCE sbom_sources_id_seq OWNED BY sbom_sources.id; +CREATE TABLE sbom_vulnerable_component_versions ( + id bigint NOT NULL, + vulnerability_advisory_id bigint, + sbom_component_version_id bigint, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL +); + +CREATE SEQUENCE sbom_vulnerable_component_versions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE sbom_vulnerable_component_versions_id_seq OWNED BY sbom_vulnerable_component_versions.id; + CREATE TABLE schema_migrations ( version character varying NOT NULL, finished_at timestamp with time zone DEFAULT now() @@ -22285,6 +22302,44 @@ CREATE SEQUENCE vulnerabilities_id_seq ALTER SEQUENCE vulnerabilities_id_seq OWNED BY vulnerabilities.id; +CREATE TABLE vulnerability_advisories ( + uuid uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + id bigint NOT NULL, + created_date date NOT NULL, + published_date date NOT NULL, + description text, + title text, + component_name text, + solution text, + not_impacted text, + cvss_v2 text, + cvss_v3 text, + affected_range text, + identifiers text[] DEFAULT '{}'::text[], + fixed_versions text[] DEFAULT '{}'::text[], + urls text[] DEFAULT '{}'::text[], + links text[] DEFAULT '{}'::text[], + CONSTRAINT check_3ab0544d19 CHECK ((char_length(title) <= 2048)), + CONSTRAINT check_3b57023409 CHECK ((char_length(affected_range) <= 32)), + CONSTRAINT check_4d5cd7be9c CHECK ((char_length(component_name) <= 2048)), + CONSTRAINT check_962f256a51 CHECK ((char_length(solution) <= 2048)), + CONSTRAINT check_aae93955fb CHECK ((char_length(cvss_v3) <= 128)), + CONSTRAINT check_b8a17497f3 CHECK ((char_length(cvss_v2) <= 128)), + CONSTRAINT check_c05a35f418 CHECK ((char_length(not_impacted) <= 2048)), + CONSTRAINT check_ff9f6483b6 CHECK ((char_length(description) <= 2048)) +); + +CREATE SEQUENCE vulnerability_advisories_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE vulnerability_advisories_id_seq OWNED BY vulnerability_advisories.id; + CREATE TABLE vulnerability_exports ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -23829,6 +23884,8 @@ ALTER TABLE ONLY sbom_occurrences ALTER COLUMN id SET DEFAULT nextval('sbom_occu ALTER TABLE ONLY sbom_sources ALTER COLUMN id SET DEFAULT nextval('sbom_sources_id_seq'::regclass); +ALTER TABLE ONLY sbom_vulnerable_component_versions ALTER COLUMN id SET DEFAULT nextval('sbom_vulnerable_component_versions_id_seq'::regclass); + ALTER TABLE ONLY scim_identities ALTER COLUMN id SET DEFAULT nextval('scim_identities_id_seq'::regclass); ALTER TABLE ONLY scim_oauth_access_tokens ALTER COLUMN id SET DEFAULT nextval('scim_oauth_access_tokens_id_seq'::regclass); @@ -23949,6 +24006,8 @@ ALTER TABLE ONLY users_statistics ALTER COLUMN id SET DEFAULT nextval('users_sta ALTER TABLE ONLY vulnerabilities ALTER COLUMN id SET DEFAULT nextval('vulnerabilities_id_seq'::regclass); +ALTER TABLE ONLY vulnerability_advisories ALTER COLUMN id SET DEFAULT nextval('vulnerability_advisories_id_seq'::regclass); + ALTER TABLE ONLY vulnerability_exports ALTER COLUMN id SET DEFAULT nextval('vulnerability_exports_id_seq'::regclass); ALTER TABLE ONLY vulnerability_external_issue_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_external_issue_links_id_seq'::regclass); @@ -26049,6 +26108,9 @@ ALTER TABLE ONLY sbom_occurrences ALTER TABLE ONLY sbom_sources ADD CONSTRAINT sbom_sources_pkey PRIMARY KEY (id); +ALTER TABLE ONLY sbom_vulnerable_component_versions + ADD CONSTRAINT sbom_vulnerable_component_versions_pkey PRIMARY KEY (id); + ALTER TABLE ONLY schema_migrations ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); @@ -26265,6 +26327,9 @@ ALTER TABLE ONLY verification_codes ALTER TABLE ONLY vulnerabilities ADD CONSTRAINT vulnerabilities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY vulnerability_advisories + ADD CONSTRAINT vulnerability_advisories_pkey PRIMARY KEY (id); + ALTER TABLE ONLY vulnerability_exports ADD CONSTRAINT vulnerability_exports_pkey PRIMARY KEY (id); @@ -30522,6 +30587,10 @@ CREATE UNIQUE INDEX index_vulnerability_statistics_on_unique_project_id ON vulne CREATE UNIQUE INDEX index_vulnerability_user_mentions_on_note_id ON vulnerability_user_mentions USING btree (note_id) WHERE (note_id IS NOT NULL); +CREATE INDEX index_vulnerable_component_versions_on_sbom_component_version ON sbom_vulnerable_component_versions USING btree (sbom_component_version_id); + +CREATE INDEX index_vulnerable_component_versions_on_vulnerability_advisory ON sbom_vulnerable_component_versions USING btree (vulnerability_advisory_id); + CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id ON vulnerability_user_mentions USING btree (vulnerability_id) WHERE (note_id IS NULL); CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id_and_note_id ON vulnerability_user_mentions USING btree (vulnerability_id, note_id); @@ -32520,6 +32589,9 @@ ALTER TABLE ONLY requirements_management_test_reports ALTER TABLE ONLY issues ADD CONSTRAINT fk_899c8f3231 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY sbom_vulnerable_component_versions + ADD CONSTRAINT fk_8a2a1197f9 FOREIGN KEY (sbom_component_version_id) REFERENCES sbom_component_versions(id) ON DELETE CASCADE; + ALTER TABLE ONLY protected_branch_merge_access_levels ADD CONSTRAINT fk_8a3072ccb3 FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE; @@ -32820,6 +32892,9 @@ ALTER TABLE ONLY lists ALTER TABLE ONLY agent_activity_events ADD CONSTRAINT fk_d6f785c9fc FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL; +ALTER TABLE ONLY sbom_vulnerable_component_versions + ADD CONSTRAINT fk_d720a1959a FOREIGN KEY (vulnerability_advisory_id) REFERENCES vulnerability_advisories(id) ON DELETE CASCADE; + ALTER TABLE ONLY metrics_users_starred_dashboards ADD CONSTRAINT fk_d76a2b9a8c FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index d317b1e9be1..99dc2b88e3e 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -5119,6 +5119,8 @@ Input type: `UpdateDependencyProxyImageTtlGroupPolicyInput` ### `Mutation.updateDependencyProxySettings` +These settings can be adjusted by the group Owner or Maintainer. However, in GitLab 16.0, we will be limiting this to the Owner role. [GitLab-#364441](https://gitlab.com/gitlab-org/gitlab/-/issues/364441) proposes making this change to match the permissions level in the user interface. + Input type: `UpdateDependencyProxySettingsInput` #### Arguments diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index a99aa277679..e9dd5409700 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -269,6 +269,13 @@ Use title case for the GitLab Container Registry. Do not use **currently** when talking about the product or its features. The documentation describes the product as it is today. ([Vale](../testing.md#vale) rule: [`CurrentStatus.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/CurrentStatus.yml)) +## default branch + +Use **default branch** to refer generically to the primary branch in the repository. +Users can set the default branch by using a UI setting. + +For examples that use the default branch, use `main` instead of [`master`](#master). + ## Dependency Proxy Use title case for the GitLab Dependency Proxy. @@ -648,7 +655,8 @@ Do not use **manpower**. Use words like **workforce** or **GitLab team members** ## master -Do not use **master**. Options are **primary** or **main**. ([Vale](../testing.md#vale) rule: [`InclusionCultural.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionCultural.yml)) +Do not use `master`. Use `main` when you need a sample [default branch name](#default-branch). +([Vale](../testing.md#vale) rule: [`InclusionCultural.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionCultural.yml)) ## may, might diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 0227dc9147c..f7458a8462d 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -488,8 +488,8 @@ the risk. End-users interact with this field, whereas GitLab automatically proce ##### Identifiers The `identifiers` array describes the detected vulnerability. An identifier object's `type` and -`value` fields are used to tell if two identifiers are the same. The user interface uses the -object's `name` and `url` fields to display the identifier. +`value` fields are used to [tell if two identifiers are the same](../../user/application_security/vulnerability_report/pipeline.md#deduplication-process). +The user interface uses the object's `name` and `url` fields to display the identifier. We recommend that you use the identifiers the GitLab scanners already define: @@ -510,11 +510,9 @@ new generic identifiers to if needed. Analyzers may also produce vendor-specific identifiers, which don't belong in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common). The first item of the `identifiers` array is called the -[primary identifier](../../user/application_security/terminology/index.md#primary-identifier). -The primary identifier is particularly important, because it is used to +[primary identifier](../../user/application_security/terminology/index.md#primary-identifier), and +it is used to [track vulnerabilities](#tracking-and-merging-vulnerabilities) as new commits are pushed to the repository. -Identifiers are also used to [merge duplicate vulnerabilities](#tracking-and-merging-vulnerabilities) -reported for the same commit, except for `CWE` and `WASC`. Not all vulnerabilities have CVEs, and a CVE can be identified multiple times. As a result, a CVE isn't a stable identifier and you shouldn't assume it as such when tracking vulnerabilities. @@ -666,11 +664,14 @@ Users may give feedback on a vulnerability: GitLab tracks vulnerabilities so that user feedback is not lost when new Git commits are pushed to the repository. -Vulnerabilities are tracked using a combination of three attributes: +Vulnerabilities are tracked using a +[`UUIDv5`](https://gitlab.com/gitlab-org/gitlab/-/blob/1272957c4a55e616569721febccb685c056ca1e4/ee/app/models/vulnerabilities/finding.rb#L364-368) +digest, which is generated by a `SHA-1` hash of four attributes: - [Report type](#category) -- [Location fingerprint](#location) - [Primary identifier](#identifiers) +- [Location fingerprint](#location) +- Project ID Right now, GitLab cannot track a vulnerability if its location changes as new Git commits are pushed, and this results in user feedback being lost. @@ -678,12 +679,7 @@ For instance, user feedback on a SAST vulnerability is lost if the affected file is renamed or the affected line moves down. This is addressed in [issue #7586](https://gitlab.com/gitlab-org/gitlab/-/issues/7586). -In some cases, the multiple scans executed in the same CI pipeline result in duplicates -that are automatically merged using the vulnerability location and identifiers. -Two vulnerabilities are considered to be the same if they share the same [location fingerprint](#location) -and at least one [identifier](#identifiers). Two identifiers are the same if they share the same `type` and `id`. -CWE and WASC identifiers are not considered because they describe categories of vulnerability flaws, -but not specific security flaws. +See also [deduplication process](../../user/application_security/vulnerability_report/pipeline.md#deduplication-process). ##### Severity and confidence diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 1ab48bd2583..80b07949f8e 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -1074,7 +1074,7 @@ Most tests for Elasticsearch logic relate to: There are some exceptions, such as checking for structural changes rather than individual records in an index. -The `:elastic_with_delete_by_query` trait was added to reduce run time for pipelines by creating and deleting indices +The `:elastic_delete_by_query` trait was added to reduce run time for pipelines by creating and deleting indices at the start and end of each context only. The [Elasticsearch DeleteByQuery API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html) is used to delete data in all indices in between examples to ensure a clean index. diff --git a/doc/user/packages/terraform_module_registry/index.md b/doc/user/packages/terraform_module_registry/index.md index 436c55f9ee0..0a3de25bf7d 100644 --- a/doc/user/packages/terraform_module_registry/index.md +++ b/doc/user/packages/terraform_module_registry/index.md @@ -24,7 +24,7 @@ When you publish a Terraform Module, if it does not exist, it is created. Prerequisites: -- A package with the same name and version must not already exist. +- A package with the same name and version must not already exist in the top-level namespace. - Your project and group names must not include a dot (`.`). For example, `source = "gitlab.example.com/my.group/project.name"`. - You must [authenticate with the API](../../../api/index.md#authentication). If authenticating with a deploy token, it must be configured with the `write_package_registry` scope. diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index d05eee7d6e6..bc53e3da441 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -479,6 +479,7 @@ sbom_components: :gitlab_main sbom_occurrences: :gitlab_main sbom_component_versions: :gitlab_main sbom_sources: :gitlab_main +sbom_vulnerable_component_versions: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -549,6 +550,7 @@ user_statuses: :gitlab_main user_synced_attributes_metadata: :gitlab_main verification_codes: :gitlab_main vulnerabilities: :gitlab_main +vulnerability_advisories: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 06e2b114c91..b4b9161f0c2 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -14,7 +14,7 @@ module Gitlab class << self def enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:query_analyzer_gitlab_schema_metrics) + Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops) end def analyze(parsed) diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index e0cb803b872..3b1751c863d 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -33,7 +33,7 @@ module Gitlab def self.enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:detect_cross_database_modification) + Feature.enabled?(:detect_cross_database_modification, type: :ops) end def self.requires_tracking?(parsed) diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index a8b51a95e59..d092cd56e46 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -206,7 +206,7 @@ module ObjectStorage def requires_multipart_upload? return false unless config.aws? - return false if use_workhorse_s3_client? && Feature.enabled?(:s3_omit_multipart_urls) + return false if use_workhorse_s3_client? !has_length end diff --git a/spec/lib/object_storage/direct_upload_spec.rb b/spec/lib/object_storage/direct_upload_spec.rb index 18a58522d12..1629aec89f5 100644 --- a/spec/lib/object_storage/direct_upload_spec.rb +++ b/spec/lib/object_storage/direct_upload_spec.rb @@ -342,84 +342,68 @@ RSpec.describe ObjectStorage::DirectUpload do context 'when length is unknown' do let(:has_length) { false } - context 'when s3_omit_multipart_urls feature flag is enabled' do - let(:consolidated_settings) { true } - - it 'omits multipart URLs' do - expect(subject).not_to have_key(:MultipartUpload) - end - - it_behaves_like 'a valid upload' - end - - context 'when s3_omit_multipart_urls feature flag is disabled' do + it_behaves_like 'a valid S3 upload with multipart data' do before do - stub_feature_flags(s3_omit_multipart_urls: false) + stub_object_storage_multipart_init(storage_url, "myUpload") end - it_behaves_like 'a valid S3 upload with multipart data' do - before do - stub_object_storage_multipart_init(storage_url, "myUpload") + context 'when maximum upload size is 0' do + let(:maximum_size) { 0 } + + it 'returns maximum number of parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(100) end - context 'when maximum upload size is 0' do - let(:maximum_size) { 0 } + it 'part size is minimum, 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end - it 'returns maximum number of parts' do - expect(subject[:MultipartUpload][:PartURLs].length).to eq(100) - end + context 'when maximum upload size is < 5 MB' do + let(:maximum_size) { 1024 } - it 'part size is minimum, 5MB' do - expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) - end + it 'returns only 1 part' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(1) end - context 'when maximum upload size is < 5 MB' do - let(:maximum_size) { 1024 } + it 'part size is minimum, 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end - it 'returns only 1 part' do - expect(subject[:MultipartUpload][:PartURLs].length).to eq(1) - end + context 'when maximum upload size is 10MB' do + let(:maximum_size) { 10.megabyte } - it 'part size is minimum, 5MB' do - expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) - end + it 'returns only 2 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(2) end - context 'when maximum upload size is 10MB' do - let(:maximum_size) { 10.megabyte } + it 'part size is minimum, 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end - it 'returns only 2 parts' do - expect(subject[:MultipartUpload][:PartURLs].length).to eq(2) - end + context 'when maximum upload size is 12MB' do + let(:maximum_size) { 12.megabyte } - it 'part size is minimum, 5MB' do - expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) - end + it 'returns only 3 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(3) end - context 'when maximum upload size is 12MB' do - let(:maximum_size) { 12.megabyte } + it 'part size is rounded-up to 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end - it 'returns only 3 parts' do - expect(subject[:MultipartUpload][:PartURLs].length).to eq(3) - end + context 'when maximum upload size is 49GB' do + let(:maximum_size) { 49.gigabyte } - it 'part size is rounded-up to 5MB' do - expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) - end + it 'returns maximum, 100 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(100) end - context 'when maximum upload size is 49GB' do - let(:maximum_size) { 49.gigabyte } - - it 'returns maximum, 100 parts' do - expect(subject[:MultipartUpload][:PartURLs].length).to eq(100) - end - - it 'part size is rounded-up to 5MB' do - expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte) - end + it 'part size is rounded-up to 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte) end end end diff --git a/spec/migrations/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions_spec.rb b/spec/migrations/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions_spec.rb new file mode 100644 index 00000000000..c53dd9de649 --- /dev/null +++ b/spec/migrations/20220819153725_add_vulnerability_advisory_foreign_key_to_sbom_vulnerable_component_versions_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "spec_helper" + +require_migration! + +RSpec.describe AddVulnerabilityAdvisoryForeignKeyToSbomVulnerableComponentVersions do + let(:table) { described_class::SOURCE_TABLE } + let(:column) { described_class::COLUMN } + let(:foreign_key) { -> { described_class.new.foreign_keys_for(table, column).first } } + + it "creates and drops the foreign key" do + reversible_migration do |migration| + migration.before -> do + expect(foreign_key.call).to be(nil) + end + + migration.after -> do + expect(foreign_key.call).to have_attributes(column: column.to_s) + end + end + end +end diff --git a/spec/migrations/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions_spec.rb b/spec/migrations/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions_spec.rb new file mode 100644 index 00000000000..b9cb6891681 --- /dev/null +++ b/spec/migrations/20220819162852_add_sbom_component_version_foreign_key_to_sbom_vulnerable_component_versions_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "spec_helper" + +require_migration! + +RSpec.describe AddSbomComponentVersionForeignKeyToSbomVulnerableComponentVersions do + let(:table) { described_class::SOURCE_TABLE } + let(:column) { described_class::COLUMN } + let(:foreign_key) { -> { described_class.new.foreign_keys_for(table, column).first } } + + it "creates and drops the foreign key" do + reversible_migration do |migration| + migration.before -> do + expect(foreign_key.call).to be(nil) + end + + migration.after -> do + expect(foreign_key.call).to have_attributes(column: column.to_s) + end + end + end +end