From 1f4988374d6c9870044a93c51e2016853193e8cf Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 22 Sep 2021 00:09:28 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- CHANGELOG.md | 4 + app/models/project.rb | 1 + config/initializers/grape_validators.rb | 1 + doc/administration/instance_limits.md | 14 ++ .../repository_storage_paths.md | 7 +- doc/api/oauth2.md | 10 ++ doc/api/project_relations_export.md | 111 +++++++++++++ doc/api/repository_files.md | 8 +- .../documentation/styleguide/word_list.md | 9 + doc/integration/oauth_provider.md | 19 +++ doc/security/rate_limits.md | 1 + .../settings/files_api_rate_limits.md | 56 +++++++ doc/user/admin_area/settings/index.md | 1 + .../settings/user_and_ip_rate_limits.md | 2 + .../application_security/policies/index.md | 29 +++- lib/api/project_export.rb | 46 ++++++ .../validators/project_portable.rb | 21 +++ package.json | 4 +- qa/Gemfile.lock | 2 +- .../5_package/helm_registry_spec.rb | 154 ++++++++++++++++++ .../5_package/maven_gradle_repository_spec.rb | 60 +------ qa/spec/spec_helper.rb | 1 + .../packages_registry_shared_context.rb | 63 +++++++ .../validators/project_portable_spec.rb | 33 ++++ spec/lib/gitlab/import_export/all_models.yml | 1 + spec/lib/rouge/formatters/html_gitlab_spec.rb | 2 +- spec/models/group_spec.rb | 1 + spec/models/project_spec.rb | 1 + spec/requests/api/project_export_spec.rb | 139 ++++++++++++++++ yarn.lock | 30 ++-- 30 files changed, 746 insertions(+), 85 deletions(-) create mode 100644 doc/api/project_relations_export.md create mode 100644 doc/user/admin_area/settings/files_api_rate_limits.md create mode 100644 lib/api/validations/validators/project_portable.rb create mode 100644 qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb create mode 100644 qa/spec/support/shared_contexts/packages_registry_shared_context.rb create mode 100644 spec/lib/api/validations/validators/project_portable_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index dd4d29fd535..20a0ac86e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2491,6 +2491,10 @@ No changes. - [Add missing metrics information](gitlab-org/gitlab@89cd7fe3b95323e635b2d73e08549b2e6153dc4d) ([merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61772/edit)) - [Track usage of the resolve UI](gitlab-org/gitlab@35c8e30fce288cecefcf2f7c0077d4608e696519) ([merge request](gitlab-org/gitlab!61654)) +## 13.12.12 (2021-09-21) + +No changes. + ## 13.12.11 (2021-09-02) No changes. diff --git a/app/models/project.rb b/app/models/project.rb index 74ffeef797e..353d9ce2657 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -233,6 +233,7 @@ class Project < ApplicationRecord has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :export_jobs, class_name: 'ProjectExportJob' + has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :project has_one :project_repository, inverse_of: :project has_one :tracing_setting, class_name: 'ProjectTracingSetting' has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting' diff --git a/config/initializers/grape_validators.rb b/config/initializers/grape_validators.rb index 07dd70822a2..1492894e1fa 100644 --- a/config/initializers/grape_validators.rb +++ b/config/initializers/grape_validators.rb @@ -10,3 +10,4 @@ Grape::Validations.register_validator(:check_assignees_count, ::API::Validations Grape::Validations.register_validator(:untrusted_regexp, ::API::Validations::Validators::UntrustedRegexp) Grape::Validations.register_validator(:email_or_email_list, ::API::Validations::Validators::EmailOrEmailList) Grape::Validations.register_validator(:iteration_id, ::API::Validations::Validators::IntegerOrCustomValue) +Grape::Validations.register_validator(:project_portable, ::API::Validations::Validators::ProjectPortable) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 24ffee088f3..682fba804a6 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -88,6 +88,20 @@ requests per user. For more information, read - **Default rate limit**: Disabled by default. +### Files API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68561) in GitLab 14.3. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the `files_api_throttling` flag](../administration/feature_flags.md). On GitLab.com, this feature is available but can be configured by GitLab.com administrators only. +The feature is not ready for production use. + +This setting limits the request rate on the Packages API per user or IP address. For more information, read +[Files API rate limits](../user/admin_area/settings/files_api_rate_limits.md). + +- **Default rate limit**: Disabled by default. + ### Import/Export > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2. diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index e7edfb9f338..6d328739f3e 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -11,9 +11,10 @@ GitLab stores [repositories](../user/project/repository/index.md) on repository storage is either: - A `gitaly_address`, which points to a [Gitaly node](gitaly/index.md). -- A `path`, which points directly to the directory where the repositories are stored. This method is - deprecated and [scheduled to be removed](https://gitlab.com/gitlab-org/gitaly/-/issues/1690) in - GitLab 14.0. +- A `path`, which points directly to the directory where the repositories are stored. GitLab + directly accessing a directory containing repositories + [is deprecated](https://gitlab.com/gitlab-org/gitaly/-/issues/1690). + GitLab should be configured to access GitLab repositories though a `gitaly_address`. GitLab allows you to define multiple repository storages to distribute the storage load between several mount points. For example: diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index abf9d7af229..fcb7477f226 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -412,6 +412,16 @@ prevent breaking changes introduced in [doorkeeper 5.0.2](https://github.com/doo Don't rely on these fields as they are slated for removal in a later release. +## Revoke a token + +To revoke a token, use the `revoke` endpoint. The API returns a 200 response code and an empty +JSON hash to indicate success. + +```ruby +parameters = 'client_id=APP_ID&client_secret=APP_SECRET&token=TOKEN' +RestClient.post 'https://gitlab.example.com/oauth/revoke', parameters +``` + ## OAuth 2.0 tokens and GitLab registries Standard OAuth 2.0 tokens support different degrees of access to GitLab diff --git a/doc/api/project_relations_export.md b/doc/api/project_relations_export.md new file mode 100644 index 00000000000..2016dcbd141 --- /dev/null +++ b/doc/api/project_relations_export.md @@ -0,0 +1,111 @@ +--- +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Project Relations Export API **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70330) in GitLab 14.4 behind the `bulk_import` [feature flag](../administration/feature_flags.md), disabled by default. + +FLAG: +On GitLab.com, this feature is available. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to +[disable the `bulk_import` flag](../administration/feature_flags.md). +The feature is not ready for production use. It is still in experimental stage and might change in the future. + +With the Project Relations Export API, you can partially export project structure. This API is +similar to [project export](project_import_export.md), +but it exports each top-level relation (for example, milestones/boards/labels) as a separate file +instead of one archive. The project relations export API is primarily used in +[group migration](../user/group/import/index.md#enable-or-disable-gitlab-group-migration) +to support group project import. + +## Schedule new export + +Start a new project relations export: + +```plaintext +POST /projects/:id/export_relations +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | ID of the project owned by the authenticated user. | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/export_relations" +``` + +```json +{ + "message": "202 Accepted" +} +``` + +## Export status + +View the status of the relations export: + +```plaintext +GET /projects/:id/export_relations/status +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | ID of the project owned by the authenticated user. | + +```shell +curl --request GET --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/export_relations/status" +``` + +The status can be one of the following: + +- `0`: `started` +- `1`: `finished` +- `-1`: `failed` + +- `0` - `started` +- `1` - `finished` +- `-1` - `failed` + +```json +[ + { + "relation": "project_badges", + "status": 1, + "error": null, + "updated_at": "2021-05-04T11:25:20.423Z" + }, + { + "relation": "boards", + "status": 1, + "error": null, + "updated_at": "2021-05-04T11:25:20.085Z" + } +] +``` + +## Export download + +Download the finished relations export: + +```plaintext +GET /projects/:id/export_relations/download +``` + +| Attribute | Type | Required | Description | +| --------------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | ID of the project owned by the authenticated user. | +| `relation` | string | yes | Name of the project top-level relation to download. | + +```shell +curl --header "PRIVATE-TOKEN: " --remote-header-name \ + --remote-name "https://gitlab.example.com/api/v4/projects/1/export_relations/download?relation=labels" +``` + +```shell +ls labels.ndjson.gz +labels.ndjson.gz +``` diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 1e33aadbc1b..4fb1a94e294 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -7,9 +7,11 @@ type: reference, api # Repository files API **(FREE)** -**CRUD for repository files** +You can fetch, create, update, and delete files in your repository with this API. +You can also [configure rate limits](../user/admin_area/settings/files_api_rate_limits.md) +for this API. -**Create, read, update, and delete repository files using this API** +## Available scopes for personal access tokens The different scopes available using [personal access tokens](../user/profile/personal_access_tokens.md) are depicted in the following table. @@ -19,8 +21,6 @@ in the following table. | `read_repository` | Allows read-access to the repository files. | | `api` | Allows read-write access to the repository files. | -> `read_repository` scope was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23534) in GitLab 11.6. - ## Get file from repository Allows you to receive information about file in repository like name, size, diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index eafe0e7a1c2..2c6aab1a3aa 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -471,6 +471,15 @@ Do not use **roles** and **permissions** interchangeably. Each user is assigned Use lowercase for **runners**. These are the agents that run CI/CD jobs. See also [GitLab Runner](#gitlab-runner) and [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/233529). +## (s) + +Do not use **(s)** to make a word optionally plural. It can slow down comprehension. For example: + +Do: Select the jobs you want. +Do not: Select the job(s) you want. + +If you can select multiples of something, then write the word as plural. + ## sanity check Do not use **sanity check**. Use **check for completeness** instead. ([Vale](../testing.md#vale) rule: [`InclusionAbleism.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionAbleism.yml)) diff --git a/doc/integration/oauth_provider.md b/doc/integration/oauth_provider.md index 5df6c4f28b7..12ce2314206 100644 --- a/doc/integration/oauth_provider.md +++ b/doc/integration/oauth_provider.md @@ -88,6 +88,25 @@ To create an application for your GitLab instance: When creating application in the **Admin Area** , you can mark it as _trusted_. The user authorization step is automatically skipped for this application. +## Expiring Access Tokens + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21745) in GitLab 14.3. + +By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and +earlier, OAuth access tokens had no expiration. + +All integrations should update to support access token refresh. + +When creating new applications, you can opt-out of expiry for backward compatibility by clearing +**Expire access tokens** when creating them. The ability to opt-out +[is deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/340848). + +Existing: + +- Applications can have expiring access tokens. Edit the application and select + **Expire access tokens** to enable them. +- Tokens must be [revoked](../api/oauth2.md#revoke-a-token) or they don't expire. + ## Authorized applications Every application you authorize with your GitLab credentials is shown diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index 6045dece0c2..a5249cb162f 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -35,6 +35,7 @@ These are rate limits you can set in the Admin Area of your instance: - [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md) - [Package registry rate limits](../user/admin_area/settings/package_registry_rate_limits.md) - [Git LFS rate limits](../user/admin_area/settings/git_lfs_rate_limits.md) +- [Files API rate limits](../user/admin_area/settings/files_api_rate_limits.md) ## Non-configurable limits diff --git a/doc/user/admin_area/settings/files_api_rate_limits.md b/doc/user/admin_area/settings/files_api_rate_limits.md new file mode 100644 index 00000000000..f91d4b2fb0c --- /dev/null +++ b/doc/user/admin_area/settings/files_api_rate_limits.md @@ -0,0 +1,56 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +--- + +# Files API rate limits **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68561) in GitLab 14.3. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it +available, ask an administrator to [enable the `files_api_throttling` flag](../../../administration/feature_flags.md). +On GitLab.com, this feature is available but can be configured by GitLab.com +administrators only. The feature is not ready for production use. + +The [Repository files API](../../../api/repository_files.md) enables you to +fetch, create, update, and delete files in your repository. To improve the security +and durability of your web application, you can enforce +[rate limits](../../../security/rate_limits.md) on this API. Any rate limits you +create for the Files API override the [general user and IP rate limits](user_and_ip_rate_limits.md). + +## Define Files API rate limits + +Rate limits for the Files API are disabled by default. When enabled, they supersede +the general user and IP rate limits for requests to the +[Repository files API](../../../api/repository_files.md). You can keep any general user +and IP rate limits already in place, and increase or decrease the rate limits +for the Files API. No other new features are provided by this override. + +Prerequisites: + +- You must have the Administrator role for your instance. +- The `files_api_throttling` feature flag must be enabled. + +To override the general user and IP rate limits for requests to the Repository files API: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Network**. +1. Expand **Files API Rate Limits**. +1. Select the check boxes for the types of rate limits you want to enable: + - **Unauthenticated API request rate limit** + - **Authenticated API request rate limit** +1. _If you enabled unauthenticated API request rate limits:_ + 1. Select the **Max unauthenticated API requests per period per IP**. + 1. Select the **Unauthenticated API rate limit period in seconds**. +1. _If you enabled authenticated API request rate limits:_ + 1. Select the **Max authenticated API requests per period per user**. + 1. Select the **Authenticated API rate limit period in seconds**. + +## Resources + +- [Rate limits](../../../security/rate_limits.md) +- [Repository files API](../../../api/repository_files.md) +- [User and IP rate limits](user_and_ip_rate_limits.md) diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index ec5f3ca812f..f0bd378413a 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -98,6 +98,7 @@ To access the default page for Admin Area settings: | [User and IP rate limits](user_and_ip_rate_limits.md) | Configure limits for web and API requests. | | [Package Registry Rate Limits](package_registry_rate_limits.md) | Configure specific limits for Packages API requests that supersede the user and IP rate limits. | | [Git LFS Rate Limits](git_lfs_rate_limits.md) | Configure specific limits for Git LFS requests that supersede the user and IP rate limits. | +| [Files API Rate Limits](files_api_rate_limits.md) | Configure specific limits for Files API requests that supersede the user and IP rate limits. | | [Outbound requests](../../../security/webhooks.md) | Allow requests to the local network from hooks and services. | | [Protected Paths](protected_paths.md) | Configure paths to be protected by Rack Attack. | | [Incident Management](../../../operations/incident_management/index.md) Limits | Limit the number of inbound alerts that can be sent to a project. | diff --git a/doc/user/admin_area/settings/user_and_ip_rate_limits.md b/doc/user/admin_area/settings/user_and_ip_rate_limits.md index 32f08801c76..c138096768e 100644 --- a/doc/user/admin_area/settings/user_and_ip_rate_limits.md +++ b/doc/user/admin_area/settings/user_and_ip_rate_limits.md @@ -189,6 +189,8 @@ The possible names are: - `throttle_unauthenticated_packages_api` - `throttle_authenticated_packages_api` - `throttle_authenticated_git_lfs` +- `throttle_unauthenticated_files_api` +- `throttle_authenticated_files_api` For example, to try out throttles for all authenticated requests to non-protected paths can be done by setting diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index bd143d8608a..4d71919299c 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -255,6 +255,10 @@ The policy editor currently only supports the YAML mode. The Rule mode is tracke The YAML file with Scan Execution Policies consists of an array of objects matching Scan Execution Policy Schema nested under the `scan_execution_policy` key. You can configure a maximum of 5 policies under the `scan_execution_policy` key. +When you save a new policy, GitLab validates its contents against [this JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/validators/json_schemas/security_orchestration_policy.json). +If you're not familiar with how to read [JSON schemas](https://json-schema.org/), +the following sections and tables provide an alternative. + | Field | Type | Possible values | Description | |-------|------|-----------------|-------------| | `scan_execution_policy` | `array` of Scan Execution Policy | | List of scan execution policies (maximum 5) | @@ -291,6 +295,8 @@ This rule enforces the defined actions and schedules a scan on the provided date #### `cluster` schema +Use this schema to define `clusters` objects in the [`schedule` rule type](#schedule-rule-type). + | Field | Type | Possible values | Description | |--------------|---------------------|--------------------------|-------------| | `containers` | `array` of `string` | | The container name that will be scanned (only the first value is currently supported). | @@ -329,7 +335,10 @@ Note the following: They will use predefined CI/CD variables defined for your project. Cluster selection with the `clusters` object is supported for the `schedule` rule type. Cluster with name provided in `clusters` object must be created and configured for the project. To be able to successfully perform the `container_scanning`/`cluster_image_scanning` scans for the cluster you must follow instructions for the [Cluster Image Scanning feature](../cluster_image_scanning/index.md#prerequisites). -Here's an example: +### Example security policies project + +You can use this example in a `.gitlab/security-policies/policy.yml`, as described in +[Security policies project](#security-policies-project). ```yaml --- @@ -398,6 +407,24 @@ In this example: - Cluster Image Scanning scan runs every 24h. The scan runs on the `production-cluster` cluster and fetches vulnerabilities from the container with the name `database` configured for deployment with the name `production-application` in the `production-namespace` namespace. +### Example for scan execution policy editor + +You can use this example in the YAML mode of the [Scan Execution Policy editor](#scan-execution-policy-editor). +It corresponds to a single object from the previous example. + +```yaml +name: Enforce Secret Detection and Container Scanning in every default branch pipeline +description: This policy enforces pipeline configuration to have a job with Secret Detection and Container Scanning scans for the default branch +enabled: true +rules: + - type: pipeline + branches: + - main +actions: + - scan: secret_detection + - scan: container_scanning +``` + ## Roadmap See the [Category Direction page](https://about.gitlab.com/direction/protect/container_network_security/) diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 4041e130f9e..5a1c33fed30 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -74,6 +74,52 @@ module API accepted! end + + resource do + before do + not_found! unless ::Feature.enabled?(:bulk_import, default_enabled: :yaml) + end + + desc 'Start relations export' do + detail 'This feature was introduced in GitLab 14.4' + end + post ':id/export_relations' do + response = ::BulkImports::ExportService.new(portable: user_project, user: current_user).execute + + if response.success? + accepted! + else + render_api_error!(message: 'Project relations export could not be started.') + end + end + + desc 'Download relations export' do + detail 'This feature was introduced in GitLab 14.4' + end + params do + requires :relation, + type: String, + project_portable: true, + desc: 'Project relation name' + end + get ':id/export_relations/download' do + export = user_project.bulk_import_exports.find_by_relation(params[:relation]) + file = export&.upload&.export_file + + if file + present_carrierwave_file!(file) + else + render_api_error!('404 Not found', 404) + end + end + + desc 'Relations export status' do + detail 'This feature was introduced in GitLab 14.4' + end + get ':id/export_relations/status' do + present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end + end end end end diff --git a/lib/api/validations/validators/project_portable.rb b/lib/api/validations/validators/project_portable.rb new file mode 100644 index 00000000000..3a7ea5ea71e --- /dev/null +++ b/lib/api/validations/validators/project_portable.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Validations + module Validators + class ProjectPortable < Grape::Validations::Base + def validate_param!(attr_name, params) + portable = params[attr_name] + + portable_relations = ::BulkImports::FileTransfer.config_for(::Project.new).portable_relations + return if portable_relations.include?(portable) + + raise Grape::Exceptions::Validation.new( + params: [@scope.full_name(attr_name)], + message: "is not portable" + ) + end + end + end + end +end diff --git a/package.json b/package.json index e7d1cc4a51c..ffc6504882d 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "@rails/ujs": "6.1.3-2", "@sentry/browser": "5.30.0", "@sourcegraph/code-host-integration": "0.0.60", - "@tiptap/core": "^2.0.0-beta.108", + "@tiptap/core": "^2.0.0-beta.110", "@tiptap/extension-blockquote": "^2.0.0-beta.15", "@tiptap/extension-bold": "^2.0.0-beta.15", "@tiptap/extension-bullet-list": "^2.0.0-beta.15", @@ -71,7 +71,7 @@ "@tiptap/extension-code-block-lowlight": "2.0.0-beta.39", "@tiptap/extension-document": "^2.0.0-beta.13", "@tiptap/extension-dropcursor": "^2.0.0-beta.19", - "@tiptap/extension-gapcursor": "^2.0.0-beta.19", + "@tiptap/extension-gapcursor": "^2.0.0-beta.20", "@tiptap/extension-hard-break": "^2.0.0-beta.16", "@tiptap/extension-heading": "^2.0.0-beta.15", "@tiptap/extension-history": "^2.0.0-beta.16", diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 8c24414555f..ded3abe74b4 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -41,7 +41,7 @@ GEM capybara-screenshot (1.0.23) capybara (>= 1.0, < 4) launchy - chemlab (0.8.0) + chemlab (0.8.1) colorize (~> 0.8) i18n (~> 1.8) rake (>= 12, < 14) diff --git a/qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb new file mode 100644 index 00000000000..96d1eb30268 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/helm_registry_spec.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package', :orchestrated, :packages, :object_storage, quarantine: { + only: { job: 'object_storage' }, + issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/341209', + type: :investigating + } do + describe 'Helm Registry' do + include Runtime::Fixtures + include_context 'packages registry qa scenario' + + let(:package_name) { 'gitlab_qa_helm' } + let(:package_version) { '1.3.7' } + let(:package_type) { 'helm' } + + let(:package_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + deploy: + image: alpine:3 + script: + - apk add helm --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing + - apk add curl + - helm create #{package_name} + - cp ./Chart.yaml #{package_name} + - helm package #{package_name} + - http_code=$(curl --write-out "%{http_code}" --request POST --form 'chart=@#{package_name}-#{package_version}.tgz' --user #{username}:#{access_token} ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/api/stable/charts --output /dev/null --silent) + - '[ $http_code = "201" ]' + only: + - "#{package_project.default_branch}" + tags: + - "runner-for-#{package_project.group.name}" + YAML + } + end + + let(:package_chart_yaml_file) do + { + file_path: "Chart.yaml", + content: + <<~EOF + apiVersion: v2 + name: #{package_name} + description: GitLab QA helm package + type: application + version: #{package_version} + appVersion: "1.16.0" + EOF + } + end + + let(:client_gitlab_ci_file) do + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + pull: + image: alpine:3 + script: + - apk add helm --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing + - helm repo add --username #{username} --password #{access_token} gitlab_qa ${CI_API_V4_URL}/projects/#{package_project.id}/packages/helm/stable + - helm repo update + - helm pull gitlab_qa/#{package_name} + only: + - "#{client_project.default_branch}" + tags: + - "runner-for-#{client_project.group.name}" + YAML + } + end + + %i[personal_access_token ci_job_token project_deploy_token].each do |authentication_token_type| + context "using a #{authentication_token_type}" do + let(:username) do + case authentication_token_type + when :personal_access_token + Runtime::User.username + when :ci_job_token + 'gitlab-ci-token' + when :project_deploy_token + project_deploy_token.username + end + end + + let(:access_token) do + case authentication_token_type + when :personal_access_token + personal_access_token + when :ci_job_token + '${CI_JOB_TOKEN}' + when :project_deploy_token + project_deploy_token.password + end + end + + it "pushes and pulls a helm chart" do + # pushing + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = package_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([package_gitlab_ci_file, package_chart_yaml_file]) + end + + package_project.visit! + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package_name) + + index.click_package(package_name) + end + + Page::Project::Packages::Show.perform do |show| + expect(show).to have_package_info(package_name, package_version) + end + + # pulling + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = client_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([client_gitlab_ci_file]) + end + + client_project.visit! + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('pull') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb index 7ad46ef480b..d879e3a70c4 100644 --- a/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/maven_gradle_repository_spec.rb @@ -9,57 +9,13 @@ module QA describe 'Maven Repository with Gradle' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures + include_context 'packages registry qa scenario' let(:group_id) { 'com.gitlab.qa' } let(:artifact_id) { 'maven_gradle' } let(:package_name) { "#{group_id}/#{artifact_id}".tr('.', '/') } let(:package_version) { '1.3.7' } - - let(:personal_access_token) { Runtime::Env.personal_access_token } - - let(:package_project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'maven-with-gradle-project' - project.initialize_with_readme = true - project.visibility = :private - end - end - - let(:client_project) do - Resource::Project.fabricate_via_api! do |client_project| - client_project.name = 'gradle_client' - client_project.initialize_with_readme = true - client_project.group = package_project.group - end - end - - let(:package) do - Resource::Package.init do |package| - package.name = package_name - package.project = package_project - end - end - - let(:runner) do - Resource::Runner.fabricate! do |runner| - runner.name = "qa-runner-#{Time.now.to_i}" - runner.tags = ["runner-for-#{package_project.group.name}"] - runner.executor = :docker - runner.token = package_project.group.runners_token - end - end - - let(:gitlab_address_with_port) do - uri = URI.parse(Runtime::Scenario.gitlab_address) - "#{uri.scheme}://#{uri.host}:#{uri.port}" - end - - let(:project_deploy_token) do - Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| - deploy_token.name = 'maven-with-gradle-deploy-token' - deploy_token.project = package_project - end - end + let(:package_type) { 'maven_gradle' } let(:package_gitlab_ci_file) do { @@ -131,18 +87,6 @@ module QA } end - before do - Flow::Login.sign_in_unless_signed_in - runner - end - - after do - runner.remove_via_api! - package.remove_via_api! - package_project.remove_via_api! - client_project.remove_via_api! - end - where(:authentication_token_type, :maven_header_name) do :personal_access_token | 'Private-Token' :ci_job_token | 'Job-Token' diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 4f0f93bf020..e25892a008f 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -17,6 +17,7 @@ QA::Runtime::AllureReport.configure! QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes) Dir[::File.join(__dir__, "support/shared_examples/*.rb")].sort.each { |f| require f } +Dir[::File.join(__dir__, "support/shared_contexts/*.rb")].sort.each { |f| require f } RSpec.configure do |config| config.include QA::Support::Matchers::EventuallyMatcher diff --git a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb new file mode 100644 index 00000000000..6e197015640 --- /dev/null +++ b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'packages registry qa scenario' do + let(:personal_access_token) { Runtime::Env.personal_access_token } + + let(:package_project) do + Resource::Project.fabricate_via_api! do |project| + project.name = "#{package_type}_package_project" + project.initialize_with_readme = true + project.visibility = :private + end + end + + let(:client_project) do + Resource::Project.fabricate_via_api! do |client_project| + client_project.name = "#{package_type}_client_project" + client_project.initialize_with_readme = true + client_project.group = package_project.group + end + end + + let(:package) do + Resource::Package.init do |package| + package.name = package_name + package.project = package_project + end + end + + let(:runner) do + Resource::Runner.fabricate! do |runner| + runner.name = "qa-runner-#{Time.now.to_i}" + runner.tags = ["runner-for-#{package_project.group.name}"] + runner.executor = :docker + runner.token = package_project.group.runners_token + end + end + + let(:gitlab_address_with_port) do + uri = URI.parse(Runtime::Scenario.gitlab_address) + "#{uri.scheme}://#{uri.host}:#{uri.port}" + end + + let(:project_deploy_token) do + Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + deploy_token.name = 'helm-package-deploy-token' + deploy_token.project = package_project + end + end + + before do + Flow::Login.sign_in_unless_signed_in + runner + end + + after do + runner.remove_via_api! + package.remove_via_api! + package_project.remove_via_api! + client_project.remove_via_api! + end + end +end diff --git a/spec/lib/api/validations/validators/project_portable_spec.rb b/spec/lib/api/validations/validators/project_portable_spec.rb new file mode 100644 index 00000000000..8c1a49d5214 --- /dev/null +++ b/spec/lib/api/validations/validators/project_portable_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Validations::Validators::ProjectPortable do + include ApiValidatorsHelpers + + let(:portable) { 'labels' } + let(:not_portable) { 'project_members' } + + subject do + described_class.new(['test'], {}, false, scope.new) + end + + context 'valid portable' do + it 'does not raise a validation error' do + expect_no_validation_error('test' => portable) + end + end + + context 'empty params' do + it 'raises a validation error' do + expect_validation_error('test' => nil) + expect_validation_error('test' => '') + end + end + + context 'not portable' do + it 'raises a validation error' do + expect_validation_error('test' => not_portable) # Sha length > 40 + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 9915e7412dc..6e6365ce33e 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -595,6 +595,7 @@ project: - pending_builds - security_scans - ci_feature_usages +- bulk_import_exports award_emoji: - awardable - user diff --git a/spec/lib/rouge/formatters/html_gitlab_spec.rb b/spec/lib/rouge/formatters/html_gitlab_spec.rb index d45c8c2a8c5..4bc9b256dce 100644 --- a/spec/lib/rouge/formatters/html_gitlab_spec.rb +++ b/spec/lib/rouge/formatters/html_gitlab_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe Rouge::Formatters::HTMLGitlab do describe '#format' do - subject { described_class.format(tokens, options) } + subject { described_class.format(tokens, **options) } let(:lang) { 'ruby' } let(:lexer) { Rouge::Lexer.find_fancy(lang) } diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index d536a0783bc..74563f77cbe 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -36,6 +36,7 @@ RSpec.describe Group do it { is_expected.to have_many(:debian_distributions).class_name('Packages::Debian::GroupDistribution').dependent(:destroy) } it { is_expected.to have_many(:daily_build_group_report_results).class_name('Ci::DailyBuildGroupReportResult') } it { is_expected.to have_many(:group_callouts).class_name('Users::GroupCallout').with_foreign_key(:group_id) } + it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') } describe '#members & #requesters' do let(:requester) { create(:user) } diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 3989ddc31e8..371496ad276 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -140,6 +140,7 @@ RSpec.describe Project, factory_default: :keep do it { is_expected.to have_many(:error_tracking_client_keys).class_name('ErrorTracking::ClientKey') } it { is_expected.to have_many(:pending_builds).class_name('Ci::PendingBuild') } it { is_expected.to have_many(:ci_feature_usages).class_name('Projects::CiFeatureUsage') } + it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') } # GitLab Pages it { is_expected.to have_many(:pages_domains) } diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb index 06f4475ef79..b9c458373a8 100644 --- a/spec/requests/api/project_export_spec.rb +++ b/spec/requests/api/project_export_spec.rb @@ -457,4 +457,143 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do end end end + + describe 'export relations' do + let(:relation) { 'labels' } + let(:download_path) { "/projects/#{project.id}/export_relations/download?relation=#{relation}" } + let(:path) { "/projects/#{project.id}/export_relations" } + + let_it_be(:status_path) { "/projects/#{project.id}/export_relations/status" } + + context 'when user is a maintainer' do + before do + project.add_maintainer(user) + end + + describe 'POST /projects/:id/export_relations' do + it 'accepts the request' do + post api(path, user) + + expect(response).to have_gitlab_http_status(:accepted) + end + + context 'when response is not success' do + it 'returns api error' do + allow_next_instance_of(BulkImports::ExportService) do |service| + allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'error', http_status: :error)) + end + + post api(path, user) + + expect(response).to have_gitlab_http_status(:error) + end + end + end + + describe 'GET /projects/:id/export_relations/download' do + let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') } + let_it_be(:upload) { create(:bulk_import_export_upload, export: export) } + + context 'when export file exists' do + it 'downloads exported project relation archive' do + upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz')) + + get api(download_path, user) + + expect(response).to have_gitlab_http_status(:ok) + expect(response.header['Content-Disposition']).to eq("attachment; filename=\"labels.ndjson.gz\"; filename*=UTF-8''labels.ndjson.gz") + end + end + + context 'when relation is not portable' do + let(:relation) { ::BulkImports::FileTransfer::ProjectConfig.new(project).skipped_relations.first } + + it_behaves_like '400 response' do + let(:request) { get api(download_path, user) } + end + end + + context 'when export file does not exist' do + it 'returns 404' do + allow(upload).to receive(:export_file).and_return(nil) + + get api(download_path, user) + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + + describe 'GET /projects/:id/export_relations/status' do + it 'returns a list of relation export statuses' do + create(:bulk_import_export, :started, project: project, relation: 'labels') + create(:bulk_import_export, :finished, project: project, relation: 'milestones') + create(:bulk_import_export, :failed, project: project, relation: 'project_badges') + + get api(status_path, user) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'project_badges') + expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1) + end + end + + context 'with bulk_import FF disabled' do + before do + stub_feature_flags(bulk_import: false) + end + + describe 'POST /projects/:id/export_relations' do + it_behaves_like '404 response' do + let(:request) { post api(path, user) } + end + end + + describe 'GET /projects/:id/export_relations/download' do + let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') } + let_it_be(:upload) { create(:bulk_import_export_upload, export: export) } + + before do + upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz')) + end + + it_behaves_like '404 response' do + let(:request) { post api(path, user) } + end + end + + describe 'GET /projects/:id/export_relations/status' do + it_behaves_like '404 response' do + let(:request) { get api(status_path, user) } + end + end + end + end + + context 'when user is a developer' do + let_it_be(:developer) { create(:user) } + + before do + project.add_developer(developer) + end + + describe 'POST /projects/:id/export_relations' do + it_behaves_like '403 response' do + let(:request) { post api(path, developer) } + end + end + + describe 'GET /projects/:id/export_relations/download' do + it_behaves_like '403 response' do + let(:request) { get api(download_path, developer) } + end + end + + describe 'GET /projects/:id/export_relations/status' do + it_behaves_like '403 response' do + let(:request) { get api(status_path, developer) } + end + end + end + end end diff --git a/yarn.lock b/yarn.lock index d0795b5e02f..c0f440277c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1467,10 +1467,10 @@ dom-accessibility-api "^0.5.1" pretty-format "^26.4.2" -"@tiptap/core@^2.0.0-beta.108": - version "2.0.0-beta.108" - resolved "https://registry.yarnpkg.com/@tiptap/core/-/core-2.0.0-beta.108.tgz#fdab0b549c6915d2e1710ecc915d219857c21eef" - integrity sha512-cUMAkiCHVQk7EYyge+ChFDLBl9SktVOrFogHnjUJxQw+r1iesXf8A6u8bqi/TCMoWQetyP7ELpscMpxNaIE/rg== +"@tiptap/core@^2.0.0-beta.110": + version "2.0.0-beta.110" + resolved "https://registry.yarnpkg.com/@tiptap/core/-/core-2.0.0-beta.110.tgz#a03413056f484b875c85b26aa2eff8b3022e014f" + integrity sha512-QWfgDxommAzv1Ed9vA1KAAvBTkdWkkZmNiQIlqlyhe/5M1YffkMfy1+P7KOA+lxN9Ft5TERGa0+Fg9mK3VX2QQ== dependencies: "@types/prosemirror-commands" "^1.0.4" "@types/prosemirror-inputrules" "^1.0.4" @@ -1484,7 +1484,7 @@ prosemirror-inputrules "^1.1.3" prosemirror-keymap "^1.1.3" prosemirror-model "^1.14.3" - prosemirror-schema-list "^1.1.5" + prosemirror-schema-list "^1.1.6" prosemirror-state "^1.3.4" prosemirror-transform "^1.3.2" prosemirror-view "^1.20.1" @@ -1563,13 +1563,13 @@ prosemirror-view "^1.20.1" tippy.js "^6.3.1" -"@tiptap/extension-gapcursor@^2.0.0-beta.19": - version "2.0.0-beta.19" - resolved "https://registry.yarnpkg.com/@tiptap/extension-gapcursor/-/extension-gapcursor-2.0.0-beta.19.tgz#6d826c240496b1a77808999d51b8917adb372cc5" - integrity sha512-GZYMR+Z45bn87CMuOHyxzTJOFoCv58mNakIBdSGX+8A+ExBFeZr/qLqxDxN3wz+LRqy7pREe5K3UxJxpsYnCzA== +"@tiptap/extension-gapcursor@^2.0.0-beta.20": + version "2.0.0-beta.20" + resolved "https://registry.yarnpkg.com/@tiptap/extension-gapcursor/-/extension-gapcursor-2.0.0-beta.20.tgz#77df6b0c4adca016e2d1a2c90e94841f812aa717" + integrity sha512-E1qSQZa8bucttGHU74la+MZzilh3pjK3amdguJUUh1biowmAjtzYQo+wJP8KGBiXyyQaiUZj6kMgXqOcnbjX4Q== dependencies: "@types/prosemirror-gapcursor" "^1.0.4" - prosemirror-gapcursor "^1.1.5" + prosemirror-gapcursor "^1.2.0" "@tiptap/extension-hard-break@^2.0.0-beta.16": version "2.0.0-beta.16" @@ -9574,10 +9574,10 @@ prosemirror-dropcursor@^1.3.2, prosemirror-dropcursor@^1.3.5: prosemirror-transform "^1.1.0" prosemirror-view "^1.1.0" -prosemirror-gapcursor@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/prosemirror-gapcursor/-/prosemirror-gapcursor-1.1.5.tgz#0c37fd6cbb1d7c46358c2e7397f8da9a8b5c6246" - integrity sha512-SjbUZq5pgsBDuV3hu8GqgIpZR5eZvGLM+gPQTqjVVYSMUCfKW3EGXTEYaLHEl1bGduwqNC95O3bZflgtAb4L6w== +prosemirror-gapcursor@^1.1.5, prosemirror-gapcursor@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/prosemirror-gapcursor/-/prosemirror-gapcursor-1.2.0.tgz#28fb60bf3d9baf1f920907d2c3e613137204e8f3" + integrity sha512-yCLy5+0rVqLir/KcHFathQj4Rf8aRHi80FmEfKtM0JmyzvwdomslLzDZ/pX4oFhFKDgjl/WBBBFNqDyNifWg7g== dependencies: prosemirror-keymap "^1.0.0" prosemirror-model "^1.0.0" @@ -9631,7 +9631,7 @@ prosemirror-schema-basic@^1.1.2: dependencies: prosemirror-model "^1.2.0" -prosemirror-schema-list@^1.1.4, prosemirror-schema-list@^1.1.5, prosemirror-schema-list@^1.1.6: +prosemirror-schema-list@^1.1.4, prosemirror-schema-list@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/prosemirror-schema-list/-/prosemirror-schema-list-1.1.6.tgz#c3e13fe2f74750e4a53ff88d798dc0c4ccca6707" integrity sha512-aFGEdaCWmJzouZ8DwedmvSsL50JpRkqhQ6tcpThwJONVVmCgI36LJHtoQ4VGZbusMavaBhXXr33zyD2IVsTlkw==