diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index e79b7033eda..0f19b7c75f5 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -1,4 +1,4 @@ - + ### Release notes diff --git a/Gemfile.lock b/Gemfile.lock index 75e5cfd5508..d728925c507 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -822,7 +822,7 @@ GEM parslet (1.8.2) peek (1.1.0) railties (>= 4.0.0) - pg (1.2.2) + pg (1.2.3) png_quantizator (0.2.1) po_to_json (1.0.1) json (>= 1.6.0) diff --git a/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue b/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue index 2b53ac9130b..be8ce832d20 100644 --- a/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue +++ b/app/assets/javascripts/pipeline_new/components/pipeline_new_form.vue @@ -14,7 +14,7 @@ import { GlSearchBoxByType, GlSprintf, } from '@gitlab/ui'; -import { s__, __ } from '~/locale'; +import { s__, __, n__ } from '~/locale'; import axios from '~/lib/utils/axios_utils'; import { redirectTo } from '~/lib/utils/url_utility'; import { VARIABLE_TYPE, FILE_TYPE } from '../constants'; @@ -29,6 +29,8 @@ export default { ), formElementClasses: 'gl-mr-3 gl-mb-3 table-section section-15', errorTitle: __('The form contains the following error:'), + warningTitle: __('The form contains the following warning:'), + maxWarningsSummary: __('%{total} warnings found: showing first %{warningsDisplayed}'), components: { GlAlert, GlButton, @@ -74,13 +76,20 @@ export default { required: false, default: () => ({}), }, + maxWarnings: { + type: Number, + required: true, + }, }, data() { return { searchTerm: '', refValue: this.refParam, variables: {}, - error: false, + error: null, + warnings: [], + totalWarnings: 0, + isWarningDismissed: false, }; }, computed: { @@ -91,6 +100,18 @@ export default { variablesLength() { return Object.keys(this.variables).length; }, + overMaxWarningsLimit() { + return this.totalWarnings > this.maxWarnings; + }, + warningsSummary() { + return n__('%d warning found:', '%d warnings found:', this.warnings.length); + }, + summaryMessage() { + return this.overMaxWarningsLimit ? this.$options.maxWarningsSummary : this.warningsSummary; + }, + shouldShowWarning() { + return this.warnings.length > 0 && !this.isWarningDismissed; + }, }, created() { if (this.variableParams) { @@ -154,8 +175,11 @@ export default { redirectTo(`${this.pipelinesPath}/${data.id}`); }) .catch(err => { - const [message] = err.response.data.base; - this.error = message; + const { errors, warnings, total_warnings: totalWarnings } = err.response.data; + const [error] = errors; + this.error = error; + this.warnings = warnings; + this.totalWarnings = totalWarnings; }); }, }, @@ -170,8 +194,37 @@ export default { :dismissible="false" variant="danger" class="gl-mb-4" + data-testid="run-pipeline-error-alert" >{{ error }} + +
+ + + + + + +

+ {{ warning }} +

+
+ { fileParam, refNames, settingsLink, + maxWarnings, } = el?.dataset; const variableParams = JSON.parse(varParam); @@ -29,6 +30,7 @@ export default () => { fileParams, refs, settingsLink, + maxWarnings: Number(maxWarnings), }, }); }, diff --git a/app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js b/app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js index 0d46bac91a5..846b1c453a1 100644 --- a/app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js +++ b/app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js @@ -102,7 +102,8 @@ export default class MergeRequestStore { this.isPipelineSkipped = this.ciStatus === 'skipped'; this.pipelineDetailedStatus = pipelineStatus; this.isPipelineActive = data.pipeline ? data.pipeline.active : false; - this.isPipelineBlocked = pipelineStatus ? pipelineStatus.group === 'manual' : false; + this.isPipelineBlocked = + data.only_allow_merge_if_pipeline_succeeds && pipelineStatus?.group === 'manual'; this.ciStatusFaviconPath = pipelineStatus ? pipelineStatus.favicon : null; this.terraformReportsPath = data.terraform_reports_path; this.testResultsPath = data.test_reports_path; diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb index 4a105c80154..c1734d2cd8a 100644 --- a/app/controllers/projects/pipelines_controller.rb +++ b/app/controllers/projects/pipelines_controller.rb @@ -78,7 +78,10 @@ class Projects::PipelinesController < Projects::ApplicationController .represent(@pipeline), status: :created else - render json: @pipeline.errors, status: :bad_request + render json: { errors: @pipeline.error_messages.map(&:content), + warnings: @pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content), + total_warnings: @pipeline.warning_messages.length }, + status: :bad_request end end end diff --git a/app/controllers/projects/static_site_editor_controller.rb b/app/controllers/projects/static_site_editor_controller.rb index 3251814bac8..e97a8db0b79 100644 --- a/app/controllers/projects/static_site_editor_controller.rb +++ b/app/controllers/projects/static_site_editor_controller.rb @@ -14,19 +14,27 @@ class Projects::StaticSiteEditorController < Projects::ApplicationController end def show - config = Gitlab::StaticSiteEditor::Config::CombinedConfig.new( - @repository, - @ref, - @path, - params[:return_url] - ) - @data = config.data + service_response = ::StaticSiteEditor::ConfigService.new( + container: project, + current_user: current_user, + params: { + ref: @ref, + path: @path, + return_url: params[:return_url] + } + ).execute + + if service_response.success? + @data = service_response.payload + else + respond_422 + end end private def assign_ref_and_path - @ref, @path = extract_ref(params[:id]) + @ref, @path = extract_ref(params.fetch(:id)) render_404 if @ref.blank? || @path.blank? end diff --git a/app/models/packages/conan/file_metadatum.rb b/app/models/packages/conan/file_metadatum.rb index e1ef62b3959..de54580e948 100644 --- a/app/models/packages/conan/file_metadatum.rb +++ b/app/models/packages/conan/file_metadatum.rb @@ -3,6 +3,9 @@ class Packages::Conan::FileMetadatum < ApplicationRecord belongs_to :package_file, inverse_of: :conan_file_metadatum + DEFAULT_PACKAGE_REVISION = '0'.freeze + DEFAULT_RECIPE_REVISION = '0'.freeze + validates :package_file, presence: true validates :recipe_revision, diff --git a/app/presenters/packages/conan/package_presenter.rb b/app/presenters/packages/conan/package_presenter.rb index a14c6b28017..df770777ad1 100644 --- a/app/presenters/packages/conan/package_presenter.rb +++ b/app/presenters/packages/conan/package_presenter.rb @@ -3,6 +3,7 @@ module Packages module Conan class PackagePresenter + include API::Helpers::Packages::Conan::ApiHelpers include API::Helpers::RelatedResourcesHelpers include Gitlab::Utils::StrongMemoize @@ -17,7 +18,10 @@ module Packages def recipe_urls map_package_files do |package_file| - build_recipe_file_url(package_file) if package_file.conan_file_metadatum.recipe_file? + next unless package_file.conan_file_metadatum.recipe_file? + + options = url_options(package_file) + recipe_file_url(options) end end @@ -31,7 +35,12 @@ module Packages map_package_files do |package_file| next unless package_file.conan_file_metadatum.package_file? && matching_reference?(package_file) - build_package_file_url(package_file) + options = url_options(package_file).merge( + conan_package_reference: package_file.conan_file_metadatum.conan_package_reference, + package_revision: package_file.conan_file_metadatum.package_revision + ) + + package_file_url(options) end end @@ -45,36 +54,21 @@ module Packages private - def build_recipe_file_url(package_file) - expose_url( - api_v4_packages_conan_v1_files_export_path( - package_name: @package.name, - package_version: @package.version, - package_username: @package.conan_metadatum.package_username, - package_channel: @package.conan_metadatum.package_channel, - recipe_revision: package_file.conan_file_metadatum.recipe_revision, - file_name: package_file.file_name - ) - ) - end - - def build_package_file_url(package_file) - expose_url( - api_v4_packages_conan_v1_files_package_path( - package_name: @package.name, - package_version: @package.version, - package_username: @package.conan_metadatum.package_username, - package_channel: @package.conan_metadatum.package_channel, - recipe_revision: package_file.conan_file_metadatum.recipe_revision, - conan_package_reference: package_file.conan_file_metadatum.conan_package_reference, - package_revision: package_file.conan_file_metadatum.package_revision, - file_name: package_file.file_name - ) - ) + def url_options(package_file) + { + package_name: @package.name, + package_version: @package.version, + package_username: @package.conan_metadatum.package_username, + package_channel: @package.conan_metadatum.package_channel, + file_name: package_file.file_name, + recipe_revision: package_file.conan_file_metadatum.recipe_revision.presence || ::Packages::Conan::FileMetadatum::DEFAULT_RECIPE_REVISION + } end def map_package_files package_files.to_a.map do |package_file| + next unless package_file.conan_file_metadatum + key = package_file.file_name value = yield(package_file) next unless key && value diff --git a/app/services/static_site_editor/config_service.rb b/app/services/static_site_editor/config_service.rb new file mode 100644 index 00000000000..987ee071976 --- /dev/null +++ b/app/services/static_site_editor/config_service.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module StaticSiteEditor + class ConfigService < ::BaseContainerService + ValidationError = Class.new(StandardError) + + def execute + @project = container + check_access! + + ServiceResponse.success(payload: data) + rescue ValidationError => e + ServiceResponse.error(message: e.message) + end + + private + + attr_reader :project + + def check_access! + unless can?(current_user, :download_code, project) + raise ValidationError, 'Insufficient permissions to read configuration' + end + end + + def data + check_for_duplicate_keys! + generated_data.merge(file_data) + end + + def generated_data + @generated_data ||= Gitlab::StaticSiteEditor::Config::GeneratedConfig.new( + project.repository, + params.fetch(:ref), + params.fetch(:path), + params[:return_url] + ).data + end + + def file_data + @file_data ||= Gitlab::StaticSiteEditor::Config::FileConfig.new.data + end + + def check_for_duplicate_keys! + duplicate_keys = generated_data.keys & file_data.keys + raise ValidationError.new("Duplicate key(s) '#{duplicate_keys}' found.") if duplicate_keys.present? + end + end +end diff --git a/app/views/projects/pipelines/new.html.haml b/app/views/projects/pipelines/new.html.haml index 726bf9af223..8369e147b4a 100644 --- a/app/views/projects/pipelines/new.html.haml +++ b/app/views/projects/pipelines/new.html.haml @@ -7,7 +7,7 @@ %hr - if Feature.enabled?(:new_pipeline_form) - #js-new-pipeline{ data: { project_id: @project.id, pipelines_path: project_pipelines_path(@project), ref_param: params[:ref] || @project.default_branch, var_param: params[:var].to_json, file_param: params[:file_var].to_json, ref_names: @project.repository.ref_names.to_json.html_safe, settings_link: project_settings_ci_cd_path(@project) } } + #js-new-pipeline{ data: { project_id: @project.id, pipelines_path: project_pipelines_path(@project), ref_param: params[:ref] || @project.default_branch, var_param: params[:var].to_json, file_param: params[:file_var].to_json, ref_names: @project.repository.ref_names.to_json.html_safe, settings_link: project_settings_ci_cd_path(@project), max_warnings: ::Gitlab::Ci::Warnings::MAX_LIMIT } } - else = form_for @pipeline, as: :pipeline, url: project_pipelines_path(@project), html: { id: "new-pipeline-form", class: "js-new-pipeline-form js-requires-input" } do |f| diff --git a/changelogs/unreleased/11679-conan-project-level.yml b/changelogs/unreleased/11679-conan-project-level.yml new file mode 100644 index 00000000000..c57abdb5461 --- /dev/null +++ b/changelogs/unreleased/11679-conan-project-level.yml @@ -0,0 +1,5 @@ +--- +title: Allow Conan packages to be scoped to project-level +merge_request: 39541 +author: +type: added diff --git a/changelogs/unreleased/20426-allow-mr-to-merge-when-pipeline-blocked-and-pipeline-must-succeed-i.yml b/changelogs/unreleased/20426-allow-mr-to-merge-when-pipeline-blocked-and-pipeline-must-succeed-i.yml new file mode 100644 index 00000000000..898cee8d5a0 --- /dev/null +++ b/changelogs/unreleased/20426-allow-mr-to-merge-when-pipeline-blocked-and-pipeline-must-succeed-i.yml @@ -0,0 +1,6 @@ +--- +title: Merge Requests are not blocked when their pipelines are waiting for manual + actions unless 'Pipeline must succeed' is checked in the settings. +merge_request: 42207 +author: +type: fixed diff --git a/changelogs/unreleased/235874-propogate-errors-api-fuzzing.yml b/changelogs/unreleased/235874-propogate-errors-api-fuzzing.yml new file mode 100644 index 00000000000..456b8e1e9fa --- /dev/null +++ b/changelogs/unreleased/235874-propogate-errors-api-fuzzing.yml @@ -0,0 +1,5 @@ +--- +title: Fail API Fuzzing CI/CD job when scanner errors +merge_request: 41616 +author: +type: fixed diff --git a/changelogs/unreleased/245277-handle-redis-for-usage-ping-sql-dump-rake-task.yml b/changelogs/unreleased/245277-handle-redis-for-usage-ping-sql-dump-rake-task.yml new file mode 100644 index 00000000000..5219de672d3 --- /dev/null +++ b/changelogs/unreleased/245277-handle-redis-for-usage-ping-sql-dump-rake-task.yml @@ -0,0 +1,5 @@ +--- +title: Updated gitlab:usage_data:dump_sql_in_yaml rake task with redis usage +merge_request: 42189 +author: +type: changed diff --git a/doc/development/README.md b/doc/development/README.md index 6a2bf278ff2..398aea55e72 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -197,6 +197,7 @@ See [database guidelines](database/index.md). - [Defining relations between files using projections](projections.md) - [Reference processing](./reference_processing.md) - [Compatibility with multiple versions of the application running at the same time](multi_version_compatibility.md) +- [Features inside `.gitlab/`](./features_inside_dot_gitlab.md) ## Other GitLab Development Kit (GDK) guides diff --git a/doc/development/features_inside_dot_gitlab.md b/doc/development/features_inside_dot_gitlab.md new file mode 100644 index 00000000000..cb883471adf --- /dev/null +++ b/doc/development/features_inside_dot_gitlab.md @@ -0,0 +1,16 @@ +# Features inside the `.gitlab/` directory + +We have implemented standard features that depend on configuration files in the `.gitlab/` directory. You can find `.gitlab/` in various GitLab repositories. +When implementing new features, please refer to these existing features to avoid conflicts: + +- [Custom Dashboards](../operations/metrics/dashboards/index.md#add-a-new-dashboard-to-your-project): `.gitlab/dashboards/`. +- [Issue Templates](../user/project/description_templates.md#creating-issue-templates): `.gitlab/issue_templates/`. +- [Merge Request Templates](../user/project/description_templates.md#creating-merge-request-templates): `.gitlab/merge_request_templates/`. +- [GitLab Kubernetes Agents](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/configuration_repository.md#layout): `.gitlab/agents/`. +- [CODEOWNERS](../user/project/code_owners.md#how-to-set-up-code-owners): `.gitlab/CODEOWNERS`. +- [Route Maps](../ci/review_apps/#route-maps): `.gitlab/route-map.yml`. +- [Customize Auto DevOps Helm Values](../topics/autodevops/customize.md#customize-values-for-helm-chart): `.gitlab/auto-deploy-values.yaml`. +- [GitLab managed apps CI/CD](../user/clusters/applications.md#usage): `.gitlab/managed-apps/config.yaml`. +- [Insights](../user/project/insights/index.md#configure-your-insights): `.gitlab/insights.yml`. +- [Service Desk Templates](../user/project/service_desk.md#using-customized-email-templates): `.gitlab/service_desk_templates/`. +- [Web IDE](../user/project/web_ide/#web-ide-configuration-file): `.gitlab/.gitlab-webide.yml`. diff --git a/doc/development/packages.md b/doc/development/packages.md index e21eff543b4..55e22d4bb5f 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -67,7 +67,7 @@ The current state of existing package registries availability is: | Repository Type | Project Level | Group Level | Instance Level | |-----------------|---------------|-------------|----------------| | Maven | Yes | Yes | Yes | -| Conan | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | Yes | +| Conan | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | Yes | | NPM | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | | NuGet | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) | No | | PyPI | Yes | No | No | @@ -87,7 +87,7 @@ Composer package naming scope is Instance Level. To avoid name conflict for instance-level endpoints you will need to define a package naming convention that gives a way to identify the project that the package belongs to. This generally involves using the project ID or full project path in the package name. See -[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention) as an example. +[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-level-remote) as an example. For group and project-level endpoints, naming can be less constrained and it will be up to the group and project members to be certain that there is no conflict between two package names. However, the system should prevent diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 31035d6f6bf..c7853be4932 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -427,9 +427,9 @@ spec itself, but the former is preferred. It takes around one second to load tests that are using `fast_spec_helper` instead of 30+ seconds in case of a regular `spec_helper`. -### `let` variables +### `subject` and `let` variables -GitLab's RSpec suite has made extensive use of `let`(along with it strict, non-lazy +GitLab's RSpec suite has made extensive use of `let`(along with its strict, non-lazy version `let!`) variables to reduce duplication. However, this sometimes [comes at the cost of clarity](https://thoughtbot.com/blog/lets-not), so we need to set some guidelines for their use going forward: @@ -448,6 +448,7 @@ so we need to set some guidelines for their use going forward: - `let!` variables should be used only in case if strict evaluation with defined order is required, otherwise `let` will suffice. Remember that `let` is lazy and won't be evaluated until it is referenced. +- Use named `subject(:name)` over un-named `subject` in examples, as this gives the subject a contextual name. ### Common test setup diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md index bafcbbd32a1..c552c44c864 100644 --- a/doc/development/testing_guide/end_to_end/beginners_guide.md +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -287,7 +287,7 @@ Note the following important points: - Our test fabricates only what it needs, when it needs it. - The issue is fabricated through the API to save time. - GitLab prefers `let()` over instance variables. See - [best practices](../best_practices.md#let-variables). + [best practices](../best_practices.md#subject-and-let-variables). - `be_closed` is not implemented in `page/project/issue/show.rb` yet, but will be implemented in the next step. diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 3fed4b46e4a..36cb49256a6 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -3,6 +3,127 @@ NOTE: **Note:** This is a tailored extension of the Best Practices [found in the testing guide](../best_practices.md). +## Link a test to its test-case issue + +Every test should have a corresponding issue in the [Quality Testcases project](https://gitlab.com/gitlab-org/quality/testcases/). +It's recommended that you reuse the issue created to plan the test. If one does not already exist you +can create the issue yourself. Alternatively, you can run the test in a pipeline that has reporting +enabled and the test-case issue reporter will automatically create a new issue. + +Whether you create a new test-case issue or one is created automatically, you will need to manually add +a `testcase` RSpec metadata tag. In most cases, a single test will be associated with a single test-case +issue ([see below for exceptions](#exceptions)). + +For example: + +```ruby +RSpec.describe 'Stage' do + describe 'General description of the feature under test' do + it 'test name', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/:issue_id' do + ... + end + + it 'another test', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/:another_issue_id' do + ... + end + end +end +``` + +### Exceptions + +Most tests are defined by a single line of a `spec` file, which is why those tests can be linked to a +single test-case issue via the `testcase` tag. + +However, some tests don't have a one-to-one relationship between a line of a `spec` file and a test-case +issue. This is because some tests are defined in a way that means a single line is associated with +multiple tests, including: + +- Parallelized tests. +- Templated tests. +- Tests in shared examples that include more than one example. + +In those and similar cases we can't assign a single `testcase` tag and so we rely on the test-case +reporter to programmatically determine the correct test-case issue based on the name and description of +the test. In such cases, the test-case reporter will automatically create a test-case issue the first time +the test runs, if no issue exists already. + +In such a case, if you create the issue yourself or want to reuse an existing issue, +you must use this [end-to-end test issue template](https://gitlab.com/gitlab-org/quality/testcases/-/blob/master/.gitlab/issue_templates/End-to-end%20Test.md) +to format the issue description. + +To illustrate, there are two tests in the shared examples in [`qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/47b17db82c38ab704a23b5ba5d296ea0c6a732c8/qa/qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb): + +```ruby +shared_examples 'only user with access pushes and merges' do + it 'unselected maintainer user fails to push' do + ... + end + + it 'selected developer user pushes and merges' do + ... + end +end +``` + +Consider the following test that includes the shared examples: + +```ruby +RSpec.describe 'Create' do + describe 'Restricted protected branch push and merge' do + context 'when only one user is allowed to merge and push to a protected branch' do + ... + it_behaves_like 'only user with access pushes and merges' + end + end +end +``` + +There would be two associated test-case issues, one for each shared example, with the following content: + +[Test 1](https://gitlab.com/gitlab-org/quality/testcases/-/issues/600): + +````markdown +```markdown +Title: browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb | Create Restricted +protected branch push and merge when only one user is allowed to merge and push to a protected +branch behaves like only user with access pushes and merges selecte... + +Description: +### Full description + +Create Restricted protected branch push and merge when only one user is allowed to merge and push +to a protected branch behaves like only user with access pushes and merges selected developer user +pushes and merges + +### File path + +./qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb + +``` +```` + +[Test 2](https://gitlab.com/gitlab-org/quality/testcases/-/issues/602): + +````markdown +```markdown +Title: browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb | Create Restricted +protected branch push and merge when only one user is allowed to merge and push to a protected +branch behaves like only user with access pushes and merges unselec... + +Description: +### Full description + +Create Restricted protected branch push and merge when only one user is allowed to merge and push +to a protected branch behaves like only user with access pushes and merges unselected maintainer +user fails to push + +### File path + +./qa/specs/features/ee/browser_ui/3_create/repository/restrict_push_protected_branch_spec.rb +``` +```` + ## Prefer API over UI The end-to-end testing framework has the ability to fabricate its resources on a case-by-case basis. diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index 4059c1960e2..a9f54b53e5a 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -8,13 +8,14 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | Tag | Description | |-----|-------------| | `:elasticsearch` | The test requires an Elasticsearch service. It is used by the [instance-level scenario](https://gitlab.com/gitlab-org/gitlab-qa#definitions) [`Test::Integration::Elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/72b62b51bdf513e2936301cb6c7c91ec27c35b4d/qa/qa/ee/scenario/test/integration/elasticsearch.rb) to include only tests that require Elasticsearch. | +| `:gitaly_cluster` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | +| `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test will also include provisioning of at least one Kubernetes cluster to test against. *This tag is often be paired with `:orchestrated`.* | +| `:only` | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. | | `:orchestrated` | The GitLab instance under test may be [configured by `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#orchestrated-tests) to be different to the default GitLab configuration, or `gitlab-qa` may launch additional services in separate Docker containers, or both. Tests tagged with `:orchestrated` are excluded when testing environments where we can't dynamically modify GitLab's configuration (for example, Staging). | | `:quarantine` | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. Note that you can also [quarantine a test only when it runs against specific environment](environment_selection.md#quarantining-a-test-for-a-specific-environment). | | `:reliable` | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. | | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. | | `:runner` | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. | -| `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. | -| `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. -| `:only` | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. | +| `:testcase` | The link to the test case issue in the [Quality Testcases project](https://gitlab.com/gitlab-org/quality/testcases/). | diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 5cdbc8ebd4f..957f84206e9 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -81,7 +81,7 @@ The repository size limit includes repository files and LFS, and does not includ For details on manually purging files, see [reducing the repository size using Git](../../project/repository/reducing_the_repo_size_using_git.md). NOTE: **Note:** -GitLab.com repository size [is set by GitLab](../../gitlab_com/index.md#repository-size-limit). +GitLab.com repository size [is set by GitLab](../../gitlab_com/index.md#account-and-limit-settings). ## Troubleshooting diff --git a/doc/user/analytics/merge_request_analytics.md b/doc/user/analytics/merge_request_analytics.md index 2c2199240a8..6a18d46fd1a 100644 --- a/doc/user/analytics/merge_request_analytics.md +++ b/doc/user/analytics/merge_request_analytics.md @@ -50,6 +50,23 @@ Data table displaying a maximum of the 100 most recent merge requests merged for ![Throughput table](img/mr_throughput_table_v13_3.png "Merge Request Analytics - Throughput table listing the 100 merge requests most recently merged") +## Filter the data + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229266) in GitLab 13.4 + +You can filter the data that is presented on the page based on the following parameters: + +- Author +- Assignees +- Labels +- Milestones + +To filter results: + +1. Click on the filter bar. +1. Select a parameter to filter by. +1. Select a value from the autocompleted results, or enter search text to refine the results. + ## Permissions The **Merge Request Analytics** feature can be accessed only: diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 9066d8a5ee4..67dd31efe2c 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -89,7 +89,7 @@ Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). | [Max number of instance level variables](../../administration/instance_limits.md#number-of-instance-level-variables) | `25` | `25` | | [Scheduled Job Archival](../../user/admin_area/settings/continuous_integration.md#archive-jobs) | 3 months | Never | -## Repository size limit +## Account and limit settings GitLab.com has the following [account limits](../admin_area/settings/account_and_limit_settings.md) enabled. If a setting is not listed, it is set to the default value. @@ -99,6 +99,7 @@ or over the repository size limit, you can [reduce your repository size with Git | Setting | GitLab.com | Default | | ----------- | ----------- | ------------- | | Repository size including LFS | 10 GB | Unlimited | +| Maximum import size | 5 GB | 50 MB | NOTE: **Note:** `git push` and GitLab project imports are limited to 5 GB per request through Cloudflare. Git LFS and imports other than a file upload are not affected by this limit. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index d3aebac3ae6..32b76cf9280 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -668,6 +668,9 @@ To enable delayed deletion of projects: 1. Expand the **Permissions, LFS, 2FA** section, and check **Enable delayed project removal**. 1. Click **Save changes**. +NOTE: **Note:** +The group setting for delayed deletion is not inherited by sub-groups and has to be individually defined for each group. + #### Prevent project forking outside group **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216987) in GitLab 13.3. diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 47bdb8d412a..7c3082e0f83 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -83,15 +83,13 @@ conan new Hello/0.1 -t Next, create a package for that recipe by running `conan create` providing the Conan user and channel: ```shell -conan create . my-org+my-group+my-project/beta +conan create . mycompany/beta ``` NOTE: **Note:** -Current [naming restrictions](#package-recipe-naming-convention) require you to name the `user` value as the `+` separated path of your project on GitLab. +If you are using the [instance level remote](#instance-level-remote), a specific [naming convention](#package-recipe-naming-convention-for-instance-level-remote) must be followed. -The example above would create a package belonging to this project: `https://gitlab.com/my-org/my-group/my-project` with a channel of `beta`. - -These two example commands generate a final package with the recipe `Hello/0.1@my-org+my-group+my-project/beta`. +These two example commands generate a final package with the recipe `Hello/0.1@mycompany/beta`. For more advanced details on creating and managing your packages, refer to the [Conan docs](https://docs.conan.io/en/latest/creating_packages.html). @@ -99,6 +97,38 @@ You are now ready to upload your package to the GitLab registry. To get started, ## Adding the GitLab Package Registry as a Conan remote +You can add the GitLab Package Registry as a Conan remote at the project or instance level. + +### Project level remote + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) in GitLab 13.4. + +The project level remote allows you to work with packages within a given project. +The advantage of using the project level remote is there are no restrictions to your +package name, however all GitLab Conan packages require a full recipe +with the user and channel (`package_name/version@user/channel`). + +To add the remote: + +```shell +conan remote add gitlab https://gitlab.example.com/api/v4/projects//packages/conan +``` + +Once the remote is set, you can use the remote when running Conan commands by adding `--remote=gitlab` to the end of your commands. + +For example: + +```shell +conan search Hello* --all --remote=gitlab +``` + +### Instance level remote + +The instance level remote allows you to use a single remote to access packages accross your entire +GitLab instance. However, when using this remote, there are certain +[package naming restrictions](#package-recipe-naming-convention-for-instance-level-remote) +that must be followed. + Add a new remote to your Conan configuration: ```shell @@ -113,6 +143,25 @@ For example: conan search 'Hello*' --remote=gitlab ``` +#### Package recipe naming convention for instance level remote + +The standard Conan recipe convention looks like `package_name/version@user/channel`, +but if you're using the [instance level remote](#instance-level-remote), the recipe +`user` must be the plus sign (`+`) separated project path. + +The following table shows some example recipes you can give your package based on +the project name and path. + +| Project | Package | Supported | +| ---------------------------------- | ----------------------------------------------- | --------- | +| `foo/bar` | `my-package/1.0.0@foo+bar/stable` | Yes | +| `foo/bar-baz/buz` | `my-package/1.0.0@foo+bar-baz+buz/stable` | Yes | +| `gitlab-org/gitlab-ce` | `my-package/1.0.0@gitlab-org+gitlab-ce/stable` | Yes | +| `gitlab-org/gitlab-ce` | `my-package/1.0.0@foo/stable` | No | + +NOTE: **Note:** +[Project level remotes](#project-level-remote) allow for more flexible package names. + ## Authenticating to the GitLab Conan Repository You need a personal access token or deploy token. @@ -142,7 +191,7 @@ Alternatively, you could explicitly include your credentials in any given comman For example: ```shell -CONAN_LOGIN_USERNAME= CONAN_PASSWORD= conan upload Hello/0.1@my-group+my-project/beta --all --remote=gitlab +CONAN_LOGIN_USERNAME= CONAN_PASSWORD= conan upload Hello/0.1@mycompany/beta --all --remote=gitlab ``` ### Setting a default remote to your project (optional) @@ -150,7 +199,7 @@ CONAN_LOGIN_USERNAME= CONAN_PASSWORD=< If you'd like Conan to always use GitLab as the registry for your package, you can tell Conan to always reference the GitLab remote for a given package recipe: ```shell -conan remote add_ref Hello/0.1@my-group+my-project/beta gitlab +conan remote add_ref Hello/0.1@mycompany/beta gitlab ``` NOTE: **Note:** @@ -165,34 +214,19 @@ The rest of the example commands in this documentation assume that you've added ## Uploading a package -First you need to [create your Conan package locally](https://docs.conan.io/en/latest/creating_packages/getting_started.html). In order to work with the GitLab Package Registry, a specific [naming convention](#package-recipe-naming-convention) must be followed. +First you need to [create your Conan package locally](https://docs.conan.io/en/latest/creating_packages/getting_started.html). + +NOTE: **Note:** +If you are using the [instance level remote](#instance-level-remote), a specific [naming convention](#package-recipe-naming-convention-for-instance-level-remote) must be followed. Ensure you have a project created on GitLab and that the personal access token you're using has the correct permissions for write access to the container registry by selecting the `api` [scope](../../../user/profile/personal_access_tokens.md#limiting-scopes-of-a-personal-access-token). You can upload your package to the GitLab Package Registry using the `conan upload` command: ```shell -conan upload Hello/0.1@my-group+my-project/beta --all +conan upload Hello/0.1@mycompany/beta --all ``` -### Package recipe naming convention - -Standard Conan recipe convention looks like `package_name/version@user/channel`. - -**The recipe user must be the `+` separated project path**. The package -name may be anything, but it is preferred that the project name be used unless -it's not possible due to a naming collision. For example: - -| Project | Package | Supported | -| ---------------------------------- | ----------------------------------------------- | --------- | -| `foo/bar` | `my-package/1.0.0@foo+bar/stable` | Yes | -| `foo/bar-baz/buz` | `my-package/1.0.0@foo+bar-baz+buz/stable` | Yes | -| `gitlab-org/gitlab-ce` | `my-package/1.0.0@gitlab-org+gitlab-ce/stable` | Yes | -| `gitlab-org/gitlab-ce` | `my-package/1.0.0@foo/stable` | No | - -NOTE: **Note:** -A future iteration will extend support to [project and group level](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) remotes which allows for more flexible naming conventions. - ## Installing a package Conan packages are commonly installed as dependencies using the `conanfile.txt` file. @@ -204,7 +238,7 @@ Add the Conan recipe to the `[requires]` section of the file: ```ini [requires] - Hello/0.1@my-group+my-project/beta + Hello/0.1@mycompany/beta [generators] cmake @@ -253,7 +287,7 @@ To search using a partial name, use the wildcard symbol `*`, which should be pla ```shell conan search Hello --all --remote=gitlab conan search He* --all --remote=gitlab -conan search Hello/0.1@my-group+my-project/beta --all --remote=gitlab +conan search Hello/0.1@mycompany/beta --all --remote=gitlab ``` The scope of your search includes all projects you have permission to access, this includes your private projects as well as all public projects. @@ -263,7 +297,7 @@ The scope of your search includes all projects you have permission to access, th The `conan info` command returns information about a given package: ```shell -conan info Hello/0.1@my-group+my-project/beta +conan info Hello/0.1@mycompany/beta ``` ## List of supported CLI commands diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 2b30fd9209a..97f3f69d676 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -302,7 +302,7 @@ Successfully installed mypypipackage-0.0.1 ## Using GitLab CI with PyPI packages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11678) in GitLab 13.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202012) in GitLab 13.4. To work with PyPI commands within [GitLab CI/CD](./../../../ci/README.md), you can use `CI_JOB_TOKEN` in place of the personal access token or deploy token in your commands. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 72111928a79..e2baac1a962 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -248,7 +248,7 @@ group. | Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | | Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | -| Create project in group | | | ✓ (3) | ✓ (3) | ✓ (3) | +| Create project in group | | | ✓ (3)(5) | ✓ (3) | ✓ (3) | | Share (invite) groups with groups | | | | | ✓ | | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | | Create/edit/delete iterations | | | ✓ | ✓ | ✓ | @@ -283,6 +283,7 @@ group. - The [instance level](admin_area/settings/visibility_and_access_controls.md#default-project-creation-protection). - The [group level](group/index.md#default-project-creation-level). 1. Does not apply to subgroups. +1. Developers can push commits to the default branch of a new project only if the [default branch protection](group/index.md#changing-the-default-branch-protection-of-a-group) is set to "Partially protected" or "Not protected". ### Subgroup permissions diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index a57806cf3ff..536cae263b8 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -190,7 +190,7 @@ updated every 15 minutes at most, so may not reflect recent activity. The displa The project size may differ slightly from one instance to another due to compression, housekeeping, and other factors. [Repository size limit](../../admin_area/settings/account_and_limit_settings.md) may be set by admins. -GitLab.com's repository size limit [is set by GitLab](../../gitlab_com/index.md#repository-size-limit). +GitLab.com's repository size limit [is set by GitLab](../../gitlab_com/index.md#account-and-limit-settings). ## Contributors diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index 7e7716569b8..2a641121931 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -252,9 +252,9 @@ When using repository cleanup, note: Repository size limits: -- Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md#repository-size-limit) +- Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md#account-and-limit-settings) on self-managed instances. **(STARTER ONLY)** -- Are [set for GitLab.com](../../gitlab_com/index.md#repository-size-limit). +- Are [set for GitLab.com](../../gitlab_com/index.md#account-and-limit-settings). When a project has reached its size limit, you cannot: diff --git a/lib/api/api.rb b/lib/api/api.rb index ba77203154e..b37751e1b47 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -194,7 +194,8 @@ module API mount ::API::NugetPackages mount ::API::PypiPackages mount ::API::ComposerPackages - mount ::API::ConanPackages + mount ::API::ConanProjectPackages + mount ::API::ConanInstancePackages mount ::API::MavenPackages mount ::API::NpmPackages mount ::API::GenericPackages diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb new file mode 100644 index 00000000000..209748d79fa --- /dev/null +++ b/lib/api/conan_instance_packages.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Conan Instance-Level Package Manager Client API +module API + class ConanInstancePackages < Grape::API::Instance + namespace 'packages/conan/v1' do + include ConanPackageEndpoints + end + end +end diff --git a/lib/api/conan_packages.rb b/lib/api/conan_package_endpoints.rb similarity index 95% rename from lib/api/conan_packages.rb rename to lib/api/conan_package_endpoints.rb index 899683de184..445447cfcd2 100644 --- a/lib/api/conan_packages.rb +++ b/lib/api/conan_package_endpoints.rb @@ -9,8 +9,8 @@ # # Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 module API - class ConanPackages < Grape::API::Instance - helpers ::API::Helpers::PackagesManagerClientsHelpers + module ConanPackageEndpoints + extend ActiveSupport::Concern PACKAGE_REQUIREMENTS = { package_name: API::NO_SLASH_URL_PART_REGEX, @@ -28,15 +28,19 @@ module API CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze - before do - require_packages_enabled! + included do + helpers ::API::Helpers::PackagesManagerClientsHelpers + helpers ::API::Helpers::Packages::Conan::ApiHelpers + helpers ::API::Helpers::RelatedResourcesHelpers - # Personal access token will be extracted from Bearer or Basic authorization - # in the overridden find_personal_access_token or find_user_from_job_token helpers - authenticate! - end + before do + require_packages_enabled! + + # Personal access token will be extracted from Bearer or Basic authorization + # in the overridden find_personal_access_token or find_user_from_job_token helpers + authenticate! + end - namespace 'packages/conan/v1' do desc 'Ping the Conan API' do detail 'This feature was introduced in GitLab 12.2' end @@ -242,7 +246,7 @@ module API delete do authorize!(:destroy_package, project) - package_event('delete_package') + package_event('delete_package', category: 'API::ConanPackages') package.destroy end @@ -341,11 +345,5 @@ module API end end end - - helpers do - include Gitlab::Utils::StrongMemoize - include ::API::Helpers::RelatedResourcesHelpers - include ::API::Helpers::Packages::Conan::ApiHelpers - end end end diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb new file mode 100644 index 00000000000..c51992231a7 --- /dev/null +++ b/lib/api/conan_project_packages.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Conan Project-Level Package Manager Client API +module API + class ConanProjectPackages < Grape::API::Instance + params do + requires :id, type: Integer, desc: 'The ID of a project', regexp: %r{\A[1-9]\d*\z} + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/conan/v1' do + include ConanPackageEndpoints + end + end + end +end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index 5b513ce1aab..dcbf933a4e1 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -5,6 +5,8 @@ module API module Packages module Conan module ApiHelpers + include Gitlab::Utils::StrongMemoize + def present_download_urls(entity) authorize!(:read_package, project) @@ -31,7 +33,7 @@ module API def recipe_upload_urls { upload_urls: Hash[ file_names.select(&method(:recipe_file?)).map do |file_name| - [file_name, recipe_file_upload_url(file_name)] + [file_name, build_recipe_file_upload_url(file_name)] end ] } end @@ -39,7 +41,7 @@ module API def package_upload_urls { upload_urls: Hash[ file_names.select(&method(:package_file?)).map do |file_name| - [file_name, package_file_upload_url(file_name)] + [file_name, build_package_file_upload_url(file_name)] end ] } end @@ -52,32 +54,58 @@ module API file_name.in?(::Packages::Conan::FileMetadatum::PACKAGE_FILES) end - def package_file_upload_url(file_name) - expose_url( - api_v4_packages_conan_v1_files_package_path( - package_name: params[:package_name], - package_version: params[:package_version], - package_username: params[:package_username], - package_channel: params[:package_channel], - recipe_revision: '0', - conan_package_reference: params[:conan_package_reference], - package_revision: '0', - file_name: file_name - ) + def build_package_file_upload_url(file_name) + options = url_options(file_name).merge( + conan_package_reference: params[:conan_package_reference], + package_revision: ::Packages::Conan::FileMetadatum::DEFAULT_PACKAGE_REVISION ) + + package_file_url(options) end - def recipe_file_upload_url(file_name) - expose_url( - api_v4_packages_conan_v1_files_export_path( - package_name: params[:package_name], - package_version: params[:package_version], - package_username: params[:package_username], - package_channel: params[:package_channel], - recipe_revision: '0', - file_name: file_name + def build_recipe_file_upload_url(file_name) + recipe_file_url(url_options(file_name)) + end + + def url_options(file_name) + { + package_name: params[:package_name], + package_version: params[:package_version], + package_username: params[:package_username], + package_channel: params[:package_channel], + file_name: file_name, + recipe_revision: ::Packages::Conan::FileMetadatum::DEFAULT_RECIPE_REVISION + } + end + + def package_file_url(options) + case package_scope + when :project + expose_url( + api_v4_projects_packages_conan_v1_files_package_path( + options.merge(id: project.id) + ) ) - ) + when :instance + expose_url( + api_v4_packages_conan_v1_files_package_path(options) + ) + end + end + + def recipe_file_url(options) + case package_scope + when :project + expose_url( + api_v4_projects_packages_conan_v1_files_export_path( + options.merge(id: project.id) + ) + ) + when :instance + expose_url( + api_v4_packages_conan_v1_files_export_path(options) + ) + end end def recipe @@ -86,8 +114,13 @@ module API def project strong_memoize(:project) do - full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username]) - Project.find_by_full_path(full_path) + case package_scope + when :project + find_project!(params[:id]) + when :instance + full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username]) + find_project!(full_path) + end end end @@ -97,6 +130,7 @@ module API .conan .with_name(params[:package_name]) .with_version(params[:package_version]) + .with_conan_username(params[:package_username]) .with_conan_channel(params[:package_channel]) .order_created .last @@ -124,7 +158,7 @@ module API conan_package_reference: params[:conan_package_reference] ).execute! - package_event('pull_package') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY + package_event('pull_package', category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY present_carrierwave_file!(package_file.file) end @@ -135,7 +169,7 @@ module API def track_push_package_event if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate - package_event('push_package') + package_event('push_package', category: 'API::ConanPackages') end end @@ -236,6 +270,10 @@ module API token end + + def package_scope + params[:id].present? ? :project : :instance + end end end end diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index a078f203cf2..c3a92b67a8b 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -118,6 +118,9 @@ apifuzzer_fuzz: # Wait for testing to complete if api fuzzer is scanning - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi # + # Propagate exit code from api fuzzer (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi + # # Run user provided pre-script - sh -c "$FUZZAPI_POST_SCRIPT" # diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3783a340257..8ace4157ad7 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -19,15 +19,15 @@ module Gitlab GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000' - NoRepository = Class.new(StandardError) - InvalidRepository = Class.new(StandardError) - InvalidBlobName = Class.new(StandardError) - InvalidRef = Class.new(StandardError) - GitError = Class.new(StandardError) - DeleteBranchError = Class.new(StandardError) - TagExistsError = Class.new(StandardError) - ChecksumError = Class.new(StandardError) - class CreateTreeError < StandardError + NoRepository = Class.new(::Gitlab::Git::BaseError) + InvalidRepository = Class.new(::Gitlab::Git::BaseError) + InvalidBlobName = Class.new(::Gitlab::Git::BaseError) + InvalidRef = Class.new(::Gitlab::Git::BaseError) + GitError = Class.new(::Gitlab::Git::BaseError) + DeleteBranchError = Class.new(::Gitlab::Git::BaseError) + TagExistsError = Class.new(::Gitlab::Git::BaseError) + ChecksumError = Class.new(::Gitlab::Git::BaseError) + class CreateTreeError < ::Gitlab::Git::BaseError attr_reader :error_code def initialize(error_code) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb index bb0c18735bb..6417ec20960 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb @@ -5,7 +5,7 @@ module Gitlab module DuplicateJobs class Client def call(worker_class, job, queue, _redis_pool, &block) - DuplicateJob.new(job, queue).schedule(&block) + ::Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(job, queue).schedule(&block) end end end diff --git a/lib/gitlab/static_site_editor/config/combined_config.rb b/lib/gitlab/static_site_editor/config/combined_config.rb deleted file mode 100644 index 387ce08e552..00000000000 --- a/lib/gitlab/static_site_editor/config/combined_config.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class CombinedConfig - def initialize(repository, ref, path, return_url) - @repository = repository - @ref = ref - @path = path - @return_url = return_url - end - - def data - generated_data = Gitlab::StaticSiteEditor::Config::GeneratedConfig.new( - @repository, - @ref, - @path, - @return_url - ).data - file_data = Gitlab::StaticSiteEditor::Config::FileConfig.new.data - check_for_duplicate_keys(generated_data, file_data) - generated_data.merge(file_data) - end - - private - - def check_for_duplicate_keys(generated_data, file_data) - duplicate_keys = generated_data.keys & file_data.keys - raise StandardError.new("Duplicate key(s) '#{duplicate_keys}' found.") if duplicate_keys.present? - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index b43a5440e6b..f647c85e1c8 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -5,9 +5,8 @@ module Gitlab module Config class FileConfig def data - merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg') { - merge_requests_illustration_path: merge_requests_illustration_path + static_site_generator: 'middleman' } end end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index 4c875b1d211..f3dce74a32f 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -14,6 +14,7 @@ module Gitlab end def data + merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg') { branch: ref, path: path, @@ -23,7 +24,8 @@ module Gitlab namespace: project.namespace.full_path, return_url: sanitize_url(return_url), is_supported_content: supported_content?.to_s, - base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path) + base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), + merge_requests_illustration_path: merge_requests_illustration_path } end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index f38050ceee6..55940f51f99 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -11,6 +11,14 @@ module Gitlab raw_sql(relation, column, :distinct) end + def redis_usage_data(counter = nil, &block) + if block_given? + { redis_usage_data_block: block.to_s } + elsif counter.present? + { redis_usage_data_counter: counter } + end + end + private def raw_sql(relation, column, distinct = nil) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index d89f004ba59..55d92bffe2b 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -305,6 +305,11 @@ msgid_plural "%d vulnerabilities dismissed" msgstr[0] "" msgstr[1] "" +msgid "%d warning found:" +msgid_plural "%d warnings found:" +msgstr[0] "" +msgstr[1] "" + msgid "%s additional commit has been omitted to prevent performance issues." msgid_plural "%s additional commits have been omitted to prevent performance issues." msgstr[0] "" @@ -819,6 +824,9 @@ msgstr "" msgid "%{total} open issue weight" msgstr "" +msgid "%{total} warnings found: showing first %{warningsDisplayed}" +msgstr "" + msgid "%{usage_ping_link_start}Learn more%{usage_ping_link_end} about what information is shared with GitLab Inc." msgstr "" @@ -25064,6 +25072,9 @@ msgstr "" msgid "The form contains the following error:" msgstr "" +msgid "The form contains the following warning:" +msgstr "" + msgid "The global settings require you to enable Two-Factor Authentication for your account." msgstr "" diff --git a/package.json b/package.json index d7b8514d9f2..9569a0d997e 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@babel/preset-env": "^7.10.1", "@gitlab/at.js": "1.5.5", "@gitlab/svgs": "1.164.0", - "@gitlab/ui": "21.2.1", + "@gitlab/ui": "21.3.0", "@gitlab/visual-review-tools": "1.6.1", "@rails/actioncable": "^6.0.3-1", "@sentry/browser": "^5.22.3", diff --git a/spec/controllers/projects/pipelines_controller_spec.rb b/spec/controllers/projects/pipelines_controller_spec.rb index 1196a702115..c3be7de25a8 100644 --- a/spec/controllers/projects/pipelines_controller_spec.rb +++ b/spec/controllers/projects/pipelines_controller_spec.rb @@ -801,6 +801,11 @@ RSpec.describe Projects::PipelinesController do context 'with an invalid .gitlab-ci.yml file' do before do stub_ci_pipeline_yaml_file(YAML.dump({ + build: { + stage: 'build', + script: 'echo', + rules: [{ when: 'always' }] + }, test: { stage: 'invalid', script: 'echo' @@ -812,9 +817,13 @@ RSpec.describe Projects::PipelinesController do expect { subject }.not_to change { project.ci_pipelines.count } expect(response).to have_gitlab_http_status(:bad_request) - expect(json_response['base']).to include( + expect(json_response['errors']).to eq([ 'test job: chosen stage does not exist; available stages are .pre, build, test, deploy, .post' + ]) + expect(json_response['warnings'][0]).to include( + 'jobs:build may allow multiple pipelines to run for a single action due to `rules:when`' ) + expect(json_response['total_warnings']).to eq(1) end end end diff --git a/spec/controllers/projects/static_site_editor_controller_spec.rb b/spec/controllers/projects/static_site_editor_controller_spec.rb index 1156887b2b6..7883c7e6f81 100644 --- a/spec/controllers/projects/static_site_editor_controller_spec.rb +++ b/spec/controllers/projects/static_site_editor_controller_spec.rb @@ -7,12 +7,6 @@ RSpec.describe Projects::StaticSiteEditorController do let_it_be(:user) { create(:user) } let(:data) { instance_double(Hash) } - before do - allow_next_instance_of(Gitlab::StaticSiteEditor::Config::CombinedConfig) do |config| - allow(config).to receive(:data) { data } - end - end - describe 'GET show' do let(:default_params) do { @@ -23,6 +17,16 @@ RSpec.describe Projects::StaticSiteEditorController do } end + let(:service_response) do + ServiceResponse.success(payload: data) + end + + before do + allow_next_instance_of(::StaticSiteEditor::ConfigService) do |instance| + allow(instance).to receive(:execute).and_return(service_response) + end + end + context 'User roles' do context 'anonymous' do before do @@ -74,6 +78,14 @@ RSpec.describe Projects::StaticSiteEditorController do expect(response).to have_gitlab_http_status(:not_found) end end + + context 'when invalid config file' do + let(:service_response) { ServiceResponse.error(message: 'invalid') } + + it 'returns 422' do + expect(response).to have_gitlab_http_status(:unprocessable_entity) + end + end end end end diff --git a/spec/features/merge_request/user_sees_merge_widget_spec.rb b/spec/features/merge_request/user_sees_merge_widget_spec.rb index c7d26dfc814..93fea44707c 100644 --- a/spec/features/merge_request/user_sees_merge_widget_spec.rb +++ b/spec/features/merge_request/user_sees_merge_widget_spec.rb @@ -127,8 +127,10 @@ RSpec.describe 'Merge request > User sees merge widget', :js do end end - context 'when merge request is in the blocked pipeline state' do + context 'when merge request is in the blocked pipeline state and pipeline must succeed' do before do + project.update_attribute(:only_allow_merge_if_pipeline_succeeds, true) + create( :ci_pipeline, project: project, diff --git a/spec/features/static_site_editor_spec.rb b/spec/features/static_site_editor_spec.rb index 9ae23b4bdec..b67e47b6ac4 100644 --- a/spec/features/static_site_editor_spec.rb +++ b/spec/features/static_site_editor_spec.rb @@ -13,7 +13,11 @@ RSpec.describe 'Static Site Editor' do visit project_show_sse_path(project, 'master/README.md') end - it 'renders Static Site Editor page' do - expect(page).to have_selector('#static-site-editor') + it 'renders Static Site Editor page with generated and file attributes' do + # assert generated config value is present + expect(page).to have_css('#static-site-editor[data-branch="master"]') + + # assert file config value is present + expect(page).to have_css('#static-site-editor[data-static-site-generator="middleman"]') end end diff --git a/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js b/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js index 8a3173a4862..97a92778f1a 100644 --- a/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js +++ b/spec/frontend/pipeline_new/components/pipeline_new_form_spec.js @@ -1,10 +1,10 @@ import { mount, shallowMount } from '@vue/test-utils'; -import { GlDropdown, GlDropdownItem, GlForm } from '@gitlab/ui'; +import { GlDropdown, GlDropdownItem, GlForm, GlSprintf } from '@gitlab/ui'; import MockAdapter from 'axios-mock-adapter'; import waitForPromises from 'helpers/wait_for_promises'; import axios from '~/lib/utils/axios_utils'; import PipelineNewForm from '~/pipeline_new/components/pipeline_new_form.vue'; -import { mockRefs, mockParams, mockPostParams, mockProjectId } from '../mock_data'; +import { mockRefs, mockParams, mockPostParams, mockProjectId, mockError } from '../mock_data'; import { redirectTo } from '~/lib/utils/url_utility'; jest.mock('~/lib/utils/url_utility', () => ({ @@ -28,6 +28,10 @@ describe('Pipeline New Form', () => { const findVariableRows = () => wrapper.findAll('[data-testid="ci-variable-row"]'); const findRemoveIcons = () => wrapper.findAll('[data-testid="remove-ci-variable-row"]'); const findKeyInputs = () => wrapper.findAll('[data-testid="pipeline-form-ci-variable-key"]'); + const findErrorAlert = () => wrapper.find('[data-testid="run-pipeline-error-alert"]'); + const findWarningAlert = () => wrapper.find('[data-testid="run-pipeline-warning-alert"]'); + const findWarningAlertSummary = () => findWarningAlert().find(GlSprintf); + const findWarnings = () => wrapper.findAll('[data-testid="run-pipeline-warning"]'); const getExpectedPostParams = () => JSON.parse(mock.history.post[0].data); const createComponent = (term = '', props = {}, method = shallowMount) => { @@ -38,6 +42,7 @@ describe('Pipeline New Form', () => { refs: mockRefs, defaultBranch: 'master', settingsLink: '', + maxWarnings: 25, ...props, }, data() { @@ -50,8 +55,6 @@ describe('Pipeline New Form', () => { beforeEach(() => { mock = new MockAdapter(axios); - - mock.onPost(pipelinesPath).reply(200, postResponse); }); afterEach(() => { @@ -62,6 +65,10 @@ describe('Pipeline New Form', () => { }); describe('Dropdown with branches and tags', () => { + beforeEach(() => { + mock.onPost(pipelinesPath).reply(200, postResponse); + }); + it('displays dropdown with all branches and tags', () => { createComponent(); expect(findDropdownItems()).toHaveLength(mockRefs.length); @@ -82,6 +89,8 @@ describe('Pipeline New Form', () => { describe('Form', () => { beforeEach(() => { createComponent('', mockParams, mount); + + mock.onPost(pipelinesPath).reply(200, postResponse); }); it('displays the correct values for the provided query params', async () => { expect(findDropdown().props('text')).toBe('tag-1'); @@ -124,4 +133,34 @@ describe('Pipeline New Form', () => { expect(findVariableRows()).toHaveLength(4); }); }); + + describe('Form errors and warnings', () => { + beforeEach(() => { + createComponent(); + + mock.onPost(pipelinesPath).reply(400, mockError); + + findForm().vm.$emit('submit', dummySubmitEvent); + + return waitForPromises(); + }); + + it('shows both error and warning', () => { + expect(findErrorAlert().exists()).toBe(true); + expect(findWarningAlert().exists()).toBe(true); + }); + + it('shows the correct error', () => { + expect(findErrorAlert().text()).toBe(mockError.errors[0]); + }); + + it('shows the correct warning title', () => { + const { length } = mockError.warnings; + expect(findWarningAlertSummary().attributes('message')).toBe(`${length} warnings found:`); + }); + + it('shows the correct amount of warnings', () => { + expect(findWarnings()).toHaveLength(mockError.warnings.length); + }); + }); }); diff --git a/spec/frontend/pipeline_new/mock_data.js b/spec/frontend/pipeline_new/mock_data.js index 55ec1fb5afc..55286e0ec7e 100644 --- a/spec/frontend/pipeline_new/mock_data.js +++ b/spec/frontend/pipeline_new/mock_data.js @@ -19,3 +19,15 @@ export const mockPostParams = { { key: 'test_file', value: 'test_file_val', variable_type: 'file' }, ], }; + +export const mockError = { + errors: [ + 'test job: chosen stage does not exist; available stages are .pre, build, test, deploy, .post', + ], + warnings: [ + 'jobs:build1 may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings', + 'jobs:build2 may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings', + 'jobs:build3 may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings', + ], + total_warnings: 7, +}; diff --git a/spec/frontend/vue_mr_widget/stores/mr_widget_store_spec.js b/spec/frontend/vue_mr_widget/stores/mr_widget_store_spec.js index 48326eda404..b691a366a0f 100644 --- a/spec/frontend/vue_mr_widget/stores/mr_widget_store_spec.js +++ b/spec/frontend/vue_mr_widget/stores/mr_widget_store_spec.js @@ -69,6 +69,38 @@ describe('MergeRequestStore', () => { }); }); + describe('isPipelineBlocked', () => { + const pipelineWaitingForManualAction = { + details: { + status: { + group: 'manual', + }, + }, + }; + + it('should be `false` when the pipeline status is missing', () => { + store.setData({ ...mockData, pipeline: undefined }); + + expect(store.isPipelineBlocked).toBe(false); + }); + + it('should be `false` when the pipeline is waiting for manual action', () => { + store.setData({ ...mockData, pipeline: pipelineWaitingForManualAction }); + + expect(store.isPipelineBlocked).toBe(false); + }); + + it('should be `true` when the pipeline is waiting for manual action and the pipeline must succeed', () => { + store.setData({ + ...mockData, + pipeline: pipelineWaitingForManualAction, + only_allow_merge_if_pipeline_succeeds: true, + }); + + expect(store.isPipelineBlocked).toBe(true); + }); + }); + describe('isNothingToMergeState', () => { it('returns true when nothingToMerge', () => { store.state = stateKey.nothingToMerge; diff --git a/spec/lib/gitlab/static_site_editor/config/combined_config_spec.rb b/spec/lib/gitlab/static_site_editor/config/combined_config_spec.rb deleted file mode 100644 index 9a31728648b..00000000000 --- a/spec/lib/gitlab/static_site_editor/config/combined_config_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::StaticSiteEditor::Config::CombinedConfig do - subject(:config) { described_class.new(repository, ref, path, return_url) } - - let(:repository) { double(:repository) } - let(:ref) { double(:ref) } - let(:path) { double(:path) } - let(:return_url) { double(:return_url) } - let(:generated_data) { { generated: true } } - let(:file_data) { { file: true } } - - describe '#data' do - subject { config.data } - - before do - allow_next_instance_of(Gitlab::StaticSiteEditor::Config::GeneratedConfig) do |config| - allow(config).to receive(:data) { generated_data } - end - allow_next_instance_of(Gitlab::StaticSiteEditor::Config::FileConfig) do |config| - allow(config).to receive(:data) { file_data } - end - end - - it 'returns merged generated data and config file data' do - is_expected.to eq({ generated: true, file: true }) - end - - it 'raises an exception if any keys would be overwritten by the merge' do - generated_data[:duplicate_key] = true - file_data[:duplicate_key] = true - expect { subject }.to raise_error(StandardError, /duplicate key.*duplicate_key.*found/i) - end - end -end diff --git a/spec/lib/gitlab/static_site_editor/config/file_config_spec.rb b/spec/lib/gitlab/static_site_editor/config/file_config_spec.rb index 665e2651d65..594425c2dab 100644 --- a/spec/lib/gitlab/static_site_editor/config/file_config_spec.rb +++ b/spec/lib/gitlab/static_site_editor/config/file_config_spec.rb @@ -9,9 +9,7 @@ RSpec.describe Gitlab::StaticSiteEditor::Config::FileConfig do subject { config.data } it 'returns hardcoded data for now' do - is_expected.to match( - merge_requests_illustration_path: %r{illustrations/merge_requests} - ) + is_expected.to match(static_site_generator: 'middleman') end end end diff --git a/spec/lib/gitlab/static_site_editor/config/generated_config_spec.rb b/spec/lib/gitlab/static_site_editor/config/generated_config_spec.rb index 44e0c9249f5..3433a54be9c 100644 --- a/spec/lib/gitlab/static_site_editor/config/generated_config_spec.rb +++ b/spec/lib/gitlab/static_site_editor/config/generated_config_spec.rb @@ -20,17 +20,19 @@ RSpec.describe Gitlab::StaticSiteEditor::Config::GeneratedConfig do subject { config.data } it 'returns data for the frontend component' do - is_expected.to eq( - branch: 'master', - commit_id: repository.commit.id, - namespace: 'namespace', - path: 'README.md', - project: 'project', - project_id: project.id, - return_url: 'http://example.com', - is_supported_content: 'true', - base_url: '/namespace/project/-/sse/master%2FREADME.md' - ) + is_expected + .to match({ + branch: 'master', + commit_id: repository.commit.id, + namespace: 'namespace', + path: 'README.md', + project: 'project', + project_id: project.id, + return_url: 'http://example.com', + is_supported_content: 'true', + base_url: '/namespace/project/-/sse/master%2FREADME.md', + merge_requests_illustration_path: %r{illustrations/merge_requests} + }) end context 'when namespace is a subgroup' do diff --git a/spec/lib/gitlab/usage_data_queries_spec.rb b/spec/lib/gitlab/usage_data_queries_spec.rb index 06f3174dd34..617c047a868 100644 --- a/spec/lib/gitlab/usage_data_queries_spec.rb +++ b/spec/lib/gitlab/usage_data_queries_spec.rb @@ -18,4 +18,18 @@ RSpec.describe Gitlab::UsageDataQueries do expect(described_class.distinct_count(Issue, :author_id)).to eq('SELECT COUNT(DISTINCT "issues"."author_id") FROM "issues"') end end + + describe '.redis_usage_data' do + subject(:redis_usage_data) { described_class.redis_usage_data { 42 } } + + it 'returns a class for redis_usage_data with a counter call' do + expect(described_class.redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)) + .to eq(redis_usage_data_counter: Gitlab::UsageDataCounters::WikiPageCounter) + end + + it 'returns a stringified block for redis_usage_data with a block' do + is_expected.to include(:redis_usage_data_block) + expect(redis_usage_data[:redis_usage_data_block]).to start_with('# "#{Settings.build_base_gitlab_url}/api/v4/projects/#{project.id}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanfile.py", + "conanmanifest.txt" => "#{Settings.build_base_gitlab_url}/api/v4/projects/#{project.id}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" + } + end + + it { is_expected.to eq(expected_result) } + end end end describe '#recipe_snapshot' do subject { described_class.new(package, user, project).recipe_snapshot } - context 'no existing package' do - let(:package) { nil } - - it { is_expected.to be_empty } - end + it_behaves_like 'no existing package' + it_behaves_like 'conan_file_metadatum is not found' context 'existing package' do let(:expected_result) do @@ -60,17 +90,21 @@ RSpec.describe ::Packages::Conan::PackagePresenter do describe '#package_urls' do let(:reference) { conan_package_reference } + let(:params) do + { + conan_package_reference: reference, + package_scope: :instance + } + end + subject do described_class.new( - package, user, project, conan_package_reference: reference + package, user, project, params ).package_urls end - context 'no existing package' do - let(:package) { nil } - - it { is_expected.to be_empty } - end + it_behaves_like 'no existing package' + it_behaves_like 'conan_file_metadatum is not found' context 'existing package' do let(:expected_result) do @@ -83,6 +117,26 @@ RSpec.describe ::Packages::Conan::PackagePresenter do it { is_expected.to eq(expected_result) } + context 'with package_scope of project' do + # #package_file_url checks for params[:id] + let(:params) do + { + conan_package_reference: reference, + id: project.id + } + end + + let(:expected_result) do + { + "conaninfo.txt" => "#{Settings.build_base_gitlab_url}/api/v4/projects/#{project.id}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/#{conan_package_reference}/0/conaninfo.txt", + "conanmanifest.txt" => "#{Settings.build_base_gitlab_url}/api/v4/projects/#{project.id}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/#{conan_package_reference}/0/conanmanifest.txt", + "conan_package.tgz" => "#{Settings.build_base_gitlab_url}/api/v4/projects/#{project.id}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/#{conan_package_reference}/0/conan_package.tgz" + } + end + + it { is_expected.to eq(expected_result) } + end + context 'multiple packages with different references' do let(:info_file) { create(:conan_package_file, :conan_package_info, package: package) } let(:manifest_file) { create(:conan_package_file, :conan_package_manifest, package: package) } @@ -132,11 +186,8 @@ RSpec.describe ::Packages::Conan::PackagePresenter do ).package_snapshot end - context 'no existing package' do - let(:package) { nil } - - it { is_expected.to be_empty } - end + it_behaves_like 'no existing package' + it_behaves_like 'conan_file_metadatum is not found' context 'existing package' do let(:expected_result) do diff --git a/spec/requests/api/conan_instance_packages_spec.rb b/spec/requests/api/conan_instance_packages_spec.rb new file mode 100644 index 00000000000..817530f0bad --- /dev/null +++ b/spec/requests/api/conan_instance_packages_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::ConanInstancePackages do + include_context 'conan api setup' + + describe 'GET /api/v4/packages/conan/v1/ping' do + let_it_be(:url) { '/packages/conan/v1/ping' } + + it_behaves_like 'conan ping endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/search' do + let_it_be(:url) { '/packages/conan/v1/conans/search' } + + it_behaves_like 'conan search endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/users/authenticate' do + let_it_be(:url) { '/packages/conan/v1/users/authenticate' } + + it_behaves_like 'conan authenticate endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/users/check_credentials' do + let_it_be(:url) { "/packages/conan/v1/users/check_credentials" } + + it_behaves_like 'conan check_credentials endpoint' + end + + context 'recipe endpoints' do + include_context 'conan recipe endpoints' + + let(:project_id) { 9999 } + let(:url_prefix) { "#{Settings.gitlab.base_url}/api/v4" } + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do + let(:recipe_path) { package.conan_recipe_path } + let(:url) { "/packages/conan/v1/conans/#{recipe_path}" } + + it_behaves_like 'recipe snapshot endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference' do + let(:recipe_path) { package.conan_recipe_path } + let(:url) { "/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}" } + + it_behaves_like 'package snapshot endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/digest' do + subject { get api("/packages/conan/v1/conans/#{recipe_path}/digest"), headers: headers } + + it_behaves_like 'recipe download_urls endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls' do + subject { get api("/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/download_urls"), headers: headers } + + it_behaves_like 'package download_urls endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/download_urls' do + subject { get api("/packages/conan/v1/conans/#{recipe_path}/download_urls"), headers: headers } + + it_behaves_like 'recipe download_urls endpoint' + end + + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/digest' do + subject { get api("/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/digest"), headers: headers } + + it_behaves_like 'package download_urls endpoint' + end + + describe 'POST /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/upload_urls' do + subject { post api("/packages/conan/v1/conans/#{recipe_path}/upload_urls"), params: params.to_json, headers: headers } + + it_behaves_like 'recipe upload_urls endpoint' + end + + describe 'POST /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/upload_urls' do + subject { post api("/packages/conan/v1/conans/#{recipe_path}/packages/123456789/upload_urls"), params: params.to_json, headers: headers } + + it_behaves_like 'package upload_urls endpoint' + end + + describe 'DELETE /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do + subject { delete api("/packages/conan/v1/conans/#{recipe_path}"), headers: headers} + + it_behaves_like 'delete package endpoint' + end + end + + context 'file download endpoints' do + include_context 'conan file download endpoints' + + describe 'GET /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ +:recipe_revision/export/:file_name' do + subject do + get api("/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/export/#{recipe_file.file_name}"), + headers: headers + end + + it_behaves_like 'recipe file download endpoint' + it_behaves_like 'project not found by recipe' + end + + describe 'GET /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ +:recipe_revision/package/:conan_package_reference/:package_revision/:file_name' do + subject do + get api("/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/package/#{metadata.conan_package_reference}/#{metadata.package_revision}/#{package_file.file_name}"), + headers: headers + end + + it_behaves_like 'package file download endpoint' + it_behaves_like 'project not found by recipe' + end + end + + context 'file upload endpoints' do + include_context 'conan file upload endpoints' + + describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name/authorize' do + let(:file_name) { 'conanfile.py' } + + subject { put api("/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}/authorize"), headers: headers_with_token } + + it_behaves_like 'workhorse authorize endpoint' + end + + describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name/authorize' do + let(:file_name) { 'conaninfo.txt' } + + subject { put api("/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}/authorize"), headers: headers_with_token } + + it_behaves_like 'workhorse authorize endpoint' + end + + describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name' do + let(:url) { "/api/v4/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}" } + + it_behaves_like 'workhorse recipe file upload endpoint' + end + + describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name' do + let(:url) { "/api/v4/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}" } + + it_behaves_like 'workhorse package file upload endpoint' + end + end +end diff --git a/spec/requests/api/conan_packages_spec.rb b/spec/requests/api/conan_packages_spec.rb deleted file mode 100644 index ab37c361618..00000000000 --- a/spec/requests/api/conan_packages_spec.rb +++ /dev/null @@ -1,986 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe API::ConanPackages do - include WorkhorseHelpers - include HttpBasicAuthHelpers - include PackagesManagerApiSpecHelpers - - let(:package) { create(:conan_package) } - let_it_be(:personal_access_token) { create(:personal_access_token) } - let_it_be(:user) { personal_access_token.user } - let(:project) { package.project } - - let(:base_secret) { SecureRandom.base64(64) } - let(:auth_token) { personal_access_token.token } - let(:job) { create(:ci_build, user: user, status: :running) } - let(:job_token) { job.token } - let(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) } - let(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) } - - let(:headers) do - { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', auth_token) } - end - - let(:jwt_secret) do - OpenSSL::HMAC.hexdigest( - OpenSSL::Digest::SHA256.new, - base_secret, - Gitlab::ConanToken::HMAC_KEY - ) - end - - before do - project.add_developer(user) - allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret) - end - - describe 'GET /api/v4/packages/conan/v1/ping' do - it 'responds with 401 Unauthorized when no token provided' do - get api('/packages/conan/v1/ping') - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'responds with 200 OK when valid token is provided' do - jwt = build_jwt(personal_access_token) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:ok) - expect(response.headers['X-Conan-Server-Capabilities']).to eq("") - end - - it 'responds with 200 OK when valid job token is provided' do - jwt = build_jwt_from_job(job) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:ok) - expect(response.headers['X-Conan-Server-Capabilities']).to eq("") - end - - it 'responds with 200 OK when valid deploy token is provided' do - jwt = build_jwt_from_deploy_token(deploy_token) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:ok) - expect(response.headers['X-Conan-Server-Capabilities']).to eq("") - end - - it 'responds with 401 Unauthorized when invalid access token ID is provided' do - jwt = build_jwt(double(id: 12345), user_id: personal_access_token.user_id) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'responds with 401 Unauthorized when invalid user is provided' do - jwt = build_jwt(personal_access_token, user_id: 12345) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'responds with 401 Unauthorized when the provided JWT is signed with different secret' do - jwt = build_jwt(personal_access_token, secret: SecureRandom.base64(32)) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'responds with 401 Unauthorized when invalid JWT is provided' do - get api('/packages/conan/v1/ping'), headers: build_token_auth_header('invalid-jwt') - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'responds with 401 Unauthorized when the job is not running' do - job.update!(status: :failed) - jwt = build_jwt_from_job(job) - get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded) - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - context 'packages feature disabled' do - it 'responds with 404 Not Found' do - stub_packages_setting(enabled: false) - get api('/packages/conan/v1/ping') - - expect(response).to have_gitlab_http_status(:not_found) - end - end - end - - describe 'GET /api/v4/packages/conan/v1/conans/search' do - before do - project.update!(visibility_level: Gitlab::VisibilityLevel::PUBLIC) - - get api('/packages/conan/v1/conans/search'), headers: headers, params: params - end - - subject { json_response['results'] } - - context 'returns packages with a matching name' do - let(:params) { { q: package.conan_recipe } } - - it { is_expected.to contain_exactly(package.conan_recipe) } - end - - context 'returns packages using a * wildcard' do - let(:params) { { q: "#{package.name[0, 3]}*" } } - - it { is_expected.to contain_exactly(package.conan_recipe) } - end - - context 'does not return non-matching packages' do - let(:params) { { q: "foo" } } - - it { is_expected.to be_blank } - end - end - - describe 'GET /api/v4/packages/conan/v1/users/authenticate' do - subject { get api('/packages/conan/v1/users/authenticate'), headers: headers } - - context 'when using invalid token' do - let(:auth_token) { 'invalid_token' } - - it 'responds with 401' do - subject - - expect(response).to have_gitlab_http_status(:unauthorized) - end - end - - context 'when valid JWT access token is provided' do - it 'responds with 200' do - subject - - expect(response).to have_gitlab_http_status(:ok) - end - - it 'token has valid validity time' do - freeze_time do - subject - - payload = JSONWebToken::HMACToken.decode( - response.body, jwt_secret).first - expect(payload['access_token']).to eq(personal_access_token.id) - expect(payload['user_id']).to eq(personal_access_token.user_id) - - duration = payload['exp'] - payload['iat'] - expect(duration).to eq(1.hour) - end - end - end - - context 'with valid job token' do - let(:auth_token) { job_token } - - it 'responds with 200' do - subject - - expect(response).to have_gitlab_http_status(:ok) - end - end - - context 'with valid deploy token' do - let(:auth_token) { deploy_token.token } - - it 'responds with 200' do - subject - - expect(response).to have_gitlab_http_status(:ok) - end - end - end - - describe 'GET /api/v4/packages/conan/v1/users/check_credentials' do - it 'responds with a 200 OK with PAT' do - get api('/packages/conan/v1/users/check_credentials'), headers: headers - - expect(response).to have_gitlab_http_status(:ok) - end - - context 'with job token' do - let(:auth_token) { job_token } - - it 'responds with a 200 OK with job token' do - get api('/packages/conan/v1/users/check_credentials'), headers: headers - - expect(response).to have_gitlab_http_status(:ok) - end - end - - context 'with deploy token' do - let(:auth_token) { deploy_token.token } - - it 'responds with a 200 OK with job token' do - get api('/packages/conan/v1/users/check_credentials'), headers: headers - - expect(response).to have_gitlab_http_status(:ok) - end - end - - it 'responds with a 401 Unauthorized when an invalid token is used' do - get api('/packages/conan/v1/users/check_credentials'), headers: build_token_auth_header('invalid-token') - - expect(response).to have_gitlab_http_status(:unauthorized) - end - end - - shared_examples 'rejects invalid recipe' do - context 'with invalid recipe path' do - let(:recipe_path) { '../../foo++../..' } - - it 'returns 400' do - subject - - expect(response).to have_gitlab_http_status(:bad_request) - end - end - end - - shared_examples 'rejects invalid file_name' do |invalid_file_name| - let(:file_name) { invalid_file_name } - - context 'with invalid file_name' do - it 'returns 400' do - subject - - expect(response).to have_gitlab_http_status(:bad_request) - end - end - end - - shared_examples 'rejects recipe for invalid project' do - context 'with invalid recipe path' do - let(:recipe_path) { 'aa/bb/not-existing-project/ccc' } - - it 'returns forbidden' do - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - end - end - - shared_examples 'rejects recipe for not found package' do - context 'with invalid recipe path' do - let(:recipe_path) do - 'aa/bb/%{project}/ccc' % { project: ::Packages::Conan::Metadatum.package_username_from(full_path: project.full_path) } - end - - it 'returns not found' do - subject - - expect(response).to have_gitlab_http_status(:not_found) - end - end - end - - shared_examples 'empty recipe for not found package' do - context 'with invalid recipe url' do - let(:recipe_path) do - 'aa/bb/%{project}/ccc' % { project: ::Packages::Conan::Metadatum.package_username_from(full_path: project.full_path) } - end - - it 'returns not found' do - allow(::Packages::Conan::PackagePresenter).to receive(:new) - .with( - nil, - user, - project, - any_args - ).and_return(presenter) - allow(presenter).to receive(:recipe_snapshot) { {} } - allow(presenter).to receive(:package_snapshot) { {} } - - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.body).to eq("{}") - end - end - end - - shared_examples 'not selecting a package with the wrong type' do - context 'with a nuget package with same name and version' do - let(:conan_username) { ::Packages::Conan::Metadatum.package_username_from(full_path: project.full_path) } - let(:wrong_package) { create(:nuget_package, name: "wrong", version: '1.0.0', project: project) } - let(:recipe_path) { "#{wrong_package.name}/#{wrong_package.version}/#{conan_username}/foo" } - - it 'calls the presenter with a nil package' do - expect(::Packages::Conan::PackagePresenter).to receive(:new) - .with(nil, user, project, any_args) - - subject - end - end - end - - shared_examples 'recipe download_urls' do - let(:recipe_path) { package.conan_recipe_path } - - it 'returns the download_urls for the recipe files' do - expected_response = { - 'conanfile.py' => "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanfile.py", - 'conanmanifest.txt' => "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" - } - - allow(presenter).to receive(:recipe_urls) { expected_response } - - subject - - expect(json_response).to eq(expected_response) - end - - it_behaves_like 'not selecting a package with the wrong type' - end - - shared_examples 'package download_urls' do - let(:recipe_path) { package.conan_recipe_path } - - it 'returns the download_urls for the package files' do - expected_response = { - 'conaninfo.txt' => "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt", - 'conanmanifest.txt' => "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conanmanifest.txt", - 'conan_package.tgz' => "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conan_package.tgz" - } - - allow(presenter).to receive(:package_urls) { expected_response } - - subject - - expect(json_response).to eq(expected_response) - end - - it_behaves_like 'not selecting a package with the wrong type' - end - - context 'recipe endpoints' do - let(:jwt) { build_jwt(personal_access_token) } - let(:headers) { build_token_auth_header(jwt.encoded) } - let(:conan_package_reference) { '123456789' } - let(:presenter) { double('::Packages::Conan::PackagePresenter') } - - before do - allow(::Packages::Conan::PackagePresenter).to receive(:new) - .with(package, user, package.project, any_args) - .and_return(presenter) - end - - shared_examples 'rejects invalid upload_url params' do - context 'with unaccepted json format' do - let(:params) { %w[foo bar] } - - it 'returns 400' do - subject - - expect(response).to have_gitlab_http_status(:bad_request) - end - end - end - - shared_examples 'successful response when using Unicorn' do - context 'on Unicorn', :unicorn do - it 'returns successfully' do - subject - - expect(response).to have_gitlab_http_status(:ok) - end - end - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do - let(:recipe_path) { package.conan_recipe_path } - - subject { get api("/packages/conan/v1/conans/#{recipe_path}"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'empty recipe for not found package' - - context 'with existing package' do - it 'returns a hash of files with their md5 hashes' do - expected_response = { - 'conanfile.py' => 'md5hash1', - 'conanmanifest.txt' => 'md5hash2' - } - - allow(presenter).to receive(:recipe_snapshot) { expected_response } - - subject - - expect(json_response).to eq(expected_response) - end - end - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference' do - let(:recipe_path) { package.conan_recipe_path } - - subject { get api("/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'empty recipe for not found package' - - context 'with existing package' do - it 'returns a hash of md5 values for the files' do - expected_response = { - 'conaninfo.txt' => "md5hash1", - 'conanmanifest.txt' => "md5hash2", - 'conan_package.tgz' => "md5hash3" - } - - allow(presenter).to receive(:package_snapshot) { expected_response } - - subject - - expect(json_response).to eq(expected_response) - end - end - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/digest' do - subject { get api("/packages/conan/v1/conans/#{recipe_path}/digest"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'recipe download_urls' - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls' do - subject { get api("/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/download_urls"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'package download_urls' - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/download_urls' do - subject { get api("/packages/conan/v1/conans/#{recipe_path}/download_urls"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'recipe download_urls' - end - - describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/digest' do - subject { get api("/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/digest"), headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects recipe for invalid project' - it_behaves_like 'package download_urls' - end - - describe 'POST /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/upload_urls' do - let(:recipe_path) { package.conan_recipe_path } - - let(:params) do - { 'conanfile.py': 24, - 'conanmanifest.txt': 123 } - end - - subject { post api("/packages/conan/v1/conans/#{recipe_path}/upload_urls"), params: params.to_json, headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid upload_url params' - it_behaves_like 'successful response when using Unicorn' - - it 'returns a set of upload urls for the files requested' do - subject - - expected_response = { - 'conanfile.py': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanfile.py", - 'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" - } - - expect(response.body).to eq(expected_response.to_json) - end - - context 'with conan_sources and conan_export files' do - let(:params) do - { 'conan_sources.tgz': 345, - 'conan_export.tgz': 234, - 'conanmanifest.txt': 123 } - end - - it 'returns upload urls for the additional files' do - subject - - expected_response = { - 'conan_sources.tgz': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_sources.tgz", - 'conan_export.tgz': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_export.tgz", - 'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" - } - - expect(response.body).to eq(expected_response.to_json) - end - end - - context 'with an invalid file' do - let(:params) do - { 'invalid_file.txt': 10, - 'conanmanifest.txt': 123 } - end - - it 'does not return the invalid file as an upload_url' do - subject - - expected_response = { - 'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" - } - - expect(response.body).to eq(expected_response.to_json) - end - end - end - - describe 'POST /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/upload_urls' do - let(:recipe_path) { package.conan_recipe_path } - - let(:params) do - { 'conaninfo.txt': 24, - 'conanmanifest.txt': 123, - 'conan_package.tgz': 523 } - end - - subject { post api("/packages/conan/v1/conans/#{recipe_path}/packages/123456789/upload_urls"), params: params.to_json, headers: headers } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid upload_url params' - it_behaves_like 'successful response when using Unicorn' - - it 'returns a set of upload urls for the files requested' do - expected_response = { - 'conaninfo.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt", - 'conanmanifest.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conanmanifest.txt", - 'conan_package.tgz': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conan_package.tgz" - } - - subject - - expect(response.body).to eq(expected_response.to_json) - end - - context 'with invalid files' do - let(:params) do - { 'conaninfo.txt': 24, - 'invalid_file.txt': 10 } - end - - it 'returns upload urls only for the valid requested files' do - expected_response = { - 'conaninfo.txt': "#{Settings.gitlab.base_url}/api/v4/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt" - } - - subject - - expect(response.body).to eq(expected_response.to_json) - end - end - end - - describe 'DELETE /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do - let(:recipe_path) { package.conan_recipe_path } - - subject { delete api("/packages/conan/v1/conans/#{recipe_path}"), headers: headers} - - it_behaves_like 'rejects invalid recipe' - - it 'returns unauthorized for users without valid permission' do - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - - context 'with delete permissions' do - before do - project.add_maintainer(user) - end - - it_behaves_like 'a gitlab tracking event', described_class.name, 'delete_package' - - it 'deletes a package' do - expect { subject }.to change { Packages::Package.count }.from(2).to(1) - end - end - end - end - - context 'file endpoints' do - let(:jwt) { build_jwt(personal_access_token) } - let(:headers) { build_token_auth_header(jwt.encoded) } - let(:recipe_path) { package.conan_recipe_path } - - shared_examples 'denies download with no token' do - context 'with no private token' do - let(:headers) { {} } - - it 'returns 400' do - subject - - expect(response).to have_gitlab_http_status(:unauthorized) - end - end - end - - shared_examples 'a public project with packages' do - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end - end - - shared_examples 'an internal project with packages' do - before do - project.team.truncate - project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL) - end - - it_behaves_like 'denies download with no token' - - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end - end - - shared_examples 'a private project with packages' do - before do - project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE) - end - - it_behaves_like 'denies download with no token' - - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end - - it 'denies download when not enough permissions' do - project.add_guest(user) - - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - end - - shared_examples 'a project is not found' do - let(:recipe_path) { 'not/package/for/project' } - - it 'returns forbidden' do - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - end - - describe 'GET /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ -:recipe_revision/export/:file_name' do - let(:recipe_file) { package.package_files.find_by(file_name: 'conanfile.py') } - let(:metadata) { recipe_file.conan_file_metadatum } - - subject do - get api("/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/export/#{recipe_file.file_name}"), - headers: headers - end - - it_behaves_like 'a public project with packages' - it_behaves_like 'an internal project with packages' - it_behaves_like 'a private project with packages' - it_behaves_like 'a project is not found' - end - - describe 'GET /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ -:recipe_revision/package/:conan_package_reference/:package_revision/:file_name' do - let(:package_file) { package.package_files.find_by(file_name: 'conaninfo.txt') } - let(:metadata) { package_file.conan_file_metadatum } - - subject do - get api("/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/package/#{metadata.conan_package_reference}/#{metadata.package_revision}/#{package_file.file_name}"), - headers: headers - end - - it_behaves_like 'a public project with packages' - it_behaves_like 'an internal project with packages' - it_behaves_like 'a private project with packages' - it_behaves_like 'a project is not found' - - context 'tracking the conan_package.tgz download' do - let(:package_file) { package.package_files.find_by(file_name: ::Packages::Conan::FileMetadatum::PACKAGE_BINARY) } - - it_behaves_like 'a gitlab tracking event', described_class.name, 'pull_package' - end - end - end - - context 'file uploads' do - let(:jwt) { build_jwt(personal_access_token) } - let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } - let(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } } - let(:headers_with_token) { build_token_auth_header(jwt.encoded).merge(workhorse_header) } - let(:recipe_path) { "foo/bar/#{project.full_path.tr('/', '+')}/baz"} - - shared_examples 'uploads a package file' do - context 'file size above maximum limit' do - before do - params['file.size'] = project.actual_limits.conan_max_file_size + 1 - end - - it 'handles as a local file' do - subject - - expect(response).to have_gitlab_http_status(:bad_request) - end - end - - context 'with object storage disabled' do - context 'without a file from workhorse' do - let(:params) { { file: nil } } - - it 'rejects the request' do - subject - - expect(response).to have_gitlab_http_status(:bad_request) - end - end - - context 'with a file' do - it_behaves_like 'package workhorse uploads' - end - - context 'without a token' do - it 'rejects request without a token' do - headers_with_token.delete('HTTP_AUTHORIZATION') - - subject - - expect(response).to have_gitlab_http_status(:unauthorized) - end - end - - context 'when params from workhorse are correct' do - it 'creates package and stores package file' do - expect { subject } - .to change { project.packages.count }.by(1) - .and change { Packages::PackageFile.count }.by(1) - - expect(response).to have_gitlab_http_status(:ok) - - package_file = project.packages.last.package_files.reload.last - expect(package_file.file_name).to eq(params[:file].original_filename) - end - - it "doesn't attempt to migrate file to object storage" do - expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) - - subject - end - end - end - - context 'with object storage enabled' do - context 'and direct upload enabled' do - let!(:fog_connection) do - stub_package_file_object_storage(direct_upload: true) - end - - let(:tmp_object) do - fog_connection.directories.new(key: 'packages').files.create( - key: "tmp/uploads/#{file_name}", - body: 'content' - ) - end - - let(:fog_file) { fog_to_uploaded_file(tmp_object) } - - ['123123', '../../123123'].each do |remote_id| - context "with invalid remote_id: #{remote_id}" do - let(:params) do - { - file: fog_file, - 'file.remote_id' => remote_id - } - end - - it 'responds with status 403' do - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - end - end - - context 'with valid remote_id' do - let(:params) do - { - file: fog_file, - 'file.remote_id' => file_name - } - end - - it 'creates package and stores package file' do - expect { subject } - .to change { project.packages.count }.by(1) - .and change { Packages::PackageFile.count }.by(1) - - expect(response).to have_gitlab_http_status(:ok) - - package_file = project.packages.last.package_files.reload.last - expect(package_file.file_name).to eq(params[:file].original_filename) - expect(package_file.file.read).to eq('content') - end - end - end - - it_behaves_like 'background upload schedules a file migration' - end - end - - shared_examples 'workhorse authorization' do - it 'authorizes posting package with a valid token' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - end - - it 'rejects request without a valid token' do - headers_with_token['HTTP_AUTHORIZATION'] = 'foo' - - subject - - expect(response).to have_gitlab_http_status(:unauthorized) - end - - it 'rejects request without a valid permission' do - project.add_guest(user) - - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - - it 'rejects requests that bypassed gitlab-workhorse' do - headers_with_token.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) - - subject - - expect(response).to have_gitlab_http_status(:forbidden) - end - - context 'when using remote storage' do - context 'when direct upload is enabled' do - before do - stub_package_file_object_storage(enabled: true, direct_upload: true) - end - - it 'responds with status 200, location of package remote store and object details' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response).not_to have_key('TempPath') - expect(json_response['RemoteObject']).to have_key('ID') - expect(json_response['RemoteObject']).to have_key('GetURL') - expect(json_response['RemoteObject']).to have_key('StoreURL') - expect(json_response['RemoteObject']).to have_key('DeleteURL') - expect(json_response['RemoteObject']).not_to have_key('MultipartUpload') - end - end - - context 'when direct upload is disabled' do - before do - stub_package_file_object_storage(enabled: true, direct_upload: false) - end - - it 'handles as a local file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) - expect(json_response['TempPath']).to eq(::Packages::PackageFileUploader.workhorse_local_upload_path) - expect(json_response['RemoteObject']).to be_nil - end - end - end - end - - describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name/authorize' do - let(:file_name) { 'conanfile.py' } - - subject { put api("/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}/authorize"), headers: headers_with_token } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid file_name', 'conanfile.py.git%2fgit-upload-pack' - it_behaves_like 'workhorse authorization' - end - - describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name/authorize' do - let(:file_name) { 'conaninfo.txt' } - - subject { put api("/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}/authorize"), headers: headers_with_token } - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid file_name', 'conaninfo.txttest' - it_behaves_like 'workhorse authorization' - end - - describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name' do - let(:file_name) { 'conanfile.py' } - let(:params) { { file: temp_file(file_name) } } - - subject do - workhorse_finalize( - "/api/v4/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}", - method: :put, - file_key: :file, - params: params, - send_rewritten_field: true, - headers: headers_with_token - ) - end - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid file_name', 'conanfile.py.git%2fgit-upload-pack' - it_behaves_like 'uploads a package file' - end - - describe 'PUT /api/v4/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name' do - let(:file_name) { 'conaninfo.txt' } - let(:params) { { file: temp_file(file_name) } } - - subject do - workhorse_finalize( - "/api/v4/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}", - method: :put, - file_key: :file, - params: params, - headers: headers_with_token, - send_rewritten_field: true - ) - end - - it_behaves_like 'rejects invalid recipe' - it_behaves_like 'rejects invalid file_name', 'conaninfo.txttest' - it_behaves_like 'uploads a package file' - - context 'tracking the conan_package.tgz upload' do - let(:file_name) { ::Packages::Conan::FileMetadatum::PACKAGE_BINARY } - - it_behaves_like 'a gitlab tracking event', described_class.name, 'push_package' - end - end - end -end diff --git a/spec/requests/api/conan_project_packages_spec.rb b/spec/requests/api/conan_project_packages_spec.rb new file mode 100644 index 00000000000..fefaf9790b1 --- /dev/null +++ b/spec/requests/api/conan_project_packages_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe API::ConanProjectPackages do + include_context 'conan api setup' + + let(:project_id) { project.id } + + describe 'GET /api/v4/projects/:id/packages/conan/v1/ping' do + let(:url) { "/projects/#{project.id}/packages/conan/v1/ping" } + + it_behaves_like 'conan ping endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/search' do + let(:url) { "/projects/#{project.id}/packages/conan/v1/conans/search" } + + it_behaves_like 'conan search endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/users/authenticate' do + let(:url) { "/projects/#{project.id}/packages/conan/v1/users/authenticate" } + + it_behaves_like 'conan authenticate endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/users/check_credentials' do + let(:url) { "/projects/#{project.id}/packages/conan/v1/users/check_credentials" } + + it_behaves_like 'conan check_credentials endpoint' + end + + context 'recipe endpoints' do + include_context 'conan recipe endpoints' + + let(:url_prefix) { "#{Settings.gitlab.base_url}/api/v4/projects/#{project_id}" } + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do + let(:recipe_path) { package.conan_recipe_path } + let(:url) { "/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}" } + + it_behaves_like 'recipe snapshot endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference' do + let(:recipe_path) { package.conan_recipe_path } + let(:url) { "/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}" } + + it_behaves_like 'package snapshot endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/digest' do + subject { get api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/digest"), headers: headers } + + it_behaves_like 'recipe download_urls endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls' do + subject { get api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/download_urls"), headers: headers } + + it_behaves_like 'package download_urls endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/download_urls' do + subject { get api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/download_urls"), headers: headers } + + it_behaves_like 'recipe download_urls endpoint' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/digest' do + subject { get api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/packages/#{conan_package_reference}/digest"), headers: headers } + + it_behaves_like 'package download_urls endpoint' + end + + describe 'POST /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/upload_urls' do + subject { post api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/upload_urls"), params: params.to_json, headers: headers } + + it_behaves_like 'recipe upload_urls endpoint' + end + + describe 'POST /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel/packages/:conan_package_reference/upload_urls' do + subject { post api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}/packages/123456789/upload_urls"), params: params.to_json, headers: headers } + + it_behaves_like 'package upload_urls endpoint' + end + + describe 'DELETE /api/v4/projects/:id/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do + subject { delete api("/projects/#{project_id}/packages/conan/v1/conans/#{recipe_path}"), headers: headers} + + it_behaves_like 'delete package endpoint' + end + end + + context 'file download endpoints' do + include_context 'conan file download endpoints' + + describe 'GET /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ +:recipe_revision/export/:file_name' do + subject do + get api("/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/export/#{recipe_file.file_name}"), + headers: headers + end + + it_behaves_like 'recipe file download endpoint' + it_behaves_like 'project not found by project id' + end + + describe 'GET /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/ +:recipe_revision/package/:conan_package_reference/:package_revision/:file_name' do + subject do + get api("/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/#{metadata.recipe_revision}/package/#{metadata.conan_package_reference}/#{metadata.package_revision}/#{package_file.file_name}"), + headers: headers + end + + it_behaves_like 'package file download endpoint' + it_behaves_like 'project not found by project id' + end + end + + context 'file upload endpoints' do + include_context 'conan file upload endpoints' + + describe 'PUT /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name/authorize' do + let(:file_name) { 'conanfile.py' } + + subject { put api("/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}/authorize"), headers: headers_with_token } + + it_behaves_like 'workhorse authorize endpoint' + end + + describe 'PUT /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name/authorize' do + let(:file_name) { 'conaninfo.txt' } + + subject { put api("/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}/authorize"), headers: headers_with_token } + + it_behaves_like 'workhorse authorize endpoint' + end + + describe 'PUT /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:file_name' do + let(:url) { "/api/v4/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/0/export/#{file_name}" } + + it_behaves_like 'workhorse recipe file upload endpoint' + end + + describe 'PUT /api/v4/projects/:id/packages/conan/v1/files/:package_name/package_version/:package_username/:package_channel/:recipe_revision/export/:conan_package_reference/:package_revision/:file_name' do + let(:url) { "/api/v4/projects/#{project_id}/packages/conan/v1/files/#{recipe_path}/0/package/123456789/0/#{file_name}" } + + it_behaves_like 'workhorse package file upload endpoint' + end + end +end diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb index df95835ae94..b4705c4cc55 100644 --- a/spec/requests/api/projects_spec.rb +++ b/spec/requests/api/projects_spec.rb @@ -49,15 +49,15 @@ end RSpec.describe API::Projects do include ProjectForksHelper - let(:user) { create(:user) } - let(:user2) { create(:user) } - let(:user3) { create(:user) } - let(:admin) { create(:admin) } - let(:project) { create(:project, :repository, namespace: user.namespace) } - let(:project2) { create(:project, namespace: user.namespace) } - let(:project_member) { create(:project_member, :developer, user: user3, project: project) } - let(:user4) { create(:user, username: 'user.with.dot') } - let(:project3) do + let_it_be(:user) { create(:user) } + let_it_be(:user2) { create(:user) } + let_it_be(:user3) { create(:user) } + let_it_be(:admin) { create(:admin) } + let_it_be(:project, reload: true) { create(:project, :repository, namespace: user.namespace) } + let_it_be(:project2, reload: true) { create(:project, namespace: user.namespace) } + let_it_be(:project_member) { create(:project_member, :developer, user: user3, project: project) } + let_it_be(:user4) { create(:user, username: 'user.with.dot') } + let_it_be(:project3, reload: true) do create(:project, :private, :repository, @@ -71,14 +71,14 @@ RSpec.describe API::Projects do snippets_enabled: false) end - let(:project_member2) do + let_it_be(:project_member2) do create(:project_member, user: user4, project: project3, access_level: ProjectMember::MAINTAINER) end - let(:project4) do + let_it_be(:project4, reload: true) do create(:project, name: 'third_project', path: 'third_project', @@ -86,6 +86,8 @@ RSpec.describe API::Projects do namespace: user4.namespace) end + let(:user_projects) { [public_project, project, project2, project3] } + shared_context 'with language detection' do let(:ruby) { create(:programming_language, name: 'Ruby') } let(:javascript) { create(:programming_language, name: 'JavaScript') } @@ -146,14 +148,7 @@ RSpec.describe API::Projects do end end - let!(:public_project) { create(:project, :public, name: 'public_project') } - - before do - project - project2 - project3 - project4 - end + let_it_be(:public_project) { create(:project, :public, name: 'public_project') } context 'when unauthenticated' do it_behaves_like 'projects response' do @@ -171,7 +166,7 @@ RSpec.describe API::Projects do it_behaves_like 'projects response' do let(:filter) { {} } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3] } + let(:projects) { user_projects } end it_behaves_like 'projects response without N + 1 queries' do @@ -257,7 +252,7 @@ RSpec.describe API::Projects do expect(response).to include_pagination_headers expect(json_response).to be_an Array - statistics = json_response.first['statistics'] + statistics = json_response.find { |p| p['id'] == project.id }['statistics'] expect(statistics).to be_present expect(statistics).to include('commit_count', 'storage_size', 'repository_size', 'wiki_size', 'lfs_objects_size', 'job_artifacts_size', 'snippets_size') end @@ -386,14 +381,14 @@ RSpec.describe API::Projects do it_behaves_like 'projects response' do let(:filter) { { id_after: project2.id } } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3].select { |p| p.id > project2.id } } + let(:projects) { user_projects.select { |p| p.id > project2.id } } end context 'regression: empty string is ignored' do it_behaves_like 'projects response' do let(:filter) { { id_after: '' } } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3] } + let(:projects) { user_projects } end end end @@ -402,14 +397,14 @@ RSpec.describe API::Projects do it_behaves_like 'projects response' do let(:filter) { { id_before: project2.id } } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3].select { |p| p.id < project2.id } } + let(:projects) { user_projects.select { |p| p.id < project2.id } } end context 'regression: empty string is ignored' do it_behaves_like 'projects response' do let(:filter) { { id_before: '' } } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3] } + let(:projects) { user_projects } end end end @@ -418,7 +413,7 @@ RSpec.describe API::Projects do it_behaves_like 'projects response' do let(:filter) { { id_before: project2.id, id_after: public_project.id } } let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3].select { |p| p.id < project2.id && p.id > public_project.id } } + let(:projects) { user_projects.select { |p| p.id < project2.id && p.id > public_project.id } } end end @@ -481,7 +476,7 @@ RSpec.describe API::Projects do expect(response).to have_gitlab_http_status(:ok) expect(response).to include_pagination_headers expect(json_response).to be_an Array - expect(json_response.first['id']).to eq(project3.id) + expect(json_response.map { |p| p['id'] }).to eq(user_projects.map(&:id).sort.reverse) end end @@ -501,7 +496,6 @@ RSpec.describe API::Projects do let(:public_project) { create(:project, :public) } before do - project_member user3.update!(starred_projects: [project, project2, project3, public_project]) end @@ -642,7 +636,6 @@ RSpec.describe API::Projects do context 'non-admin user' do let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3] } it 'returns projects ordered normally' do get api('/projects', current_user), params: { order_by: order_by } @@ -650,7 +643,7 @@ RSpec.describe API::Projects do expect(response).to have_gitlab_http_status(:ok) expect(response).to include_pagination_headers expect(json_response).to be_an Array - expect(json_response.map { |project| project['id'] }).to eq(projects.map(&:id).reverse) + expect(json_response.map { |project| project['id'] }).to eq(user_projects.map(&:id).sort.reverse) end end end @@ -686,7 +679,8 @@ RSpec.describe API::Projects do context 'with keyset pagination' do let(:current_user) { user } - let(:projects) { [public_project, project, project2, project3] } + let(:first_project_id) { user_projects.map(&:id).min } + let(:last_project_id) { user_projects.map(&:id).max } context 'headers and records' do let(:params) { { pagination: 'keyset', order_by: :id, sort: :asc, per_page: 1 } } @@ -696,11 +690,11 @@ RSpec.describe API::Projects do expect(response.header).to include('Links') expect(response.header['Links']).to include('pagination=keyset') - expect(response.header['Links']).to include("id_after=#{public_project.id}") + expect(response.header['Links']).to include("id_after=#{first_project_id}") expect(response.header).to include('Link') expect(response.header['Link']).to include('pagination=keyset') - expect(response.header['Link']).to include("id_after=#{public_project.id}") + expect(response.header['Link']).to include("id_after=#{first_project_id}") end it 'contains only the first project with per_page = 1' do @@ -708,7 +702,7 @@ RSpec.describe API::Projects do expect(response).to have_gitlab_http_status(:ok) expect(json_response).to be_an Array - expect(json_response.map { |p| p['id'] }).to contain_exactly(public_project.id) + expect(json_response.map { |p| p['id'] }).to contain_exactly(first_project_id) end it 'still includes a link if the end has reached and there is no more data after this page' do @@ -752,11 +746,11 @@ RSpec.describe API::Projects do expect(response.header).to include('Links') expect(response.header['Links']).to include('pagination=keyset') - expect(response.header['Links']).to include("id_before=#{project3.id}") + expect(response.header['Links']).to include("id_before=#{last_project_id}") expect(response.header).to include('Link') expect(response.header['Link']).to include('pagination=keyset') - expect(response.header['Link']).to include("id_before=#{project3.id}") + expect(response.header['Link']).to include("id_before=#{last_project_id}") end it 'contains only the last project with per_page = 1' do @@ -764,7 +758,7 @@ RSpec.describe API::Projects do expect(response).to have_gitlab_http_status(:ok) expect(json_response).to be_an Array - expect(json_response.map { |p| p['id'] }).to contain_exactly(project3.id) + expect(json_response.map { |p| p['id'] }).to contain_exactly(last_project_id) end end @@ -793,7 +787,7 @@ RSpec.describe API::Projects do ids += Gitlab::Json.parse(response.body).map { |p| p['id'] } end - expect(ids).to contain_exactly(*projects.map(&:id)) + expect(ids).to contain_exactly(*user_projects.map(&:id)) end end end @@ -814,7 +808,7 @@ RSpec.describe API::Projects do .to change { Project.count }.by(1) expect(response).to have_gitlab_http_status(:created) - project = Project.first + project = Project.last expect(project.name).to eq('Foo Project') expect(project.path).to eq('foo-project') @@ -825,7 +819,7 @@ RSpec.describe API::Projects do .to change { Project.count }.by(1) expect(response).to have_gitlab_http_status(:created) - project = Project.first + project = Project.last expect(project.name).to eq('foo_project') expect(project.path).to eq('foo_project') @@ -836,7 +830,7 @@ RSpec.describe API::Projects do .to change { Project.count }.by(1) expect(response).to have_gitlab_http_status(:created) - project = Project.first + project = Project.last expect(project.name).to eq('Foo Project') expect(project.path).to eq('path-project-Foo') @@ -1985,7 +1979,8 @@ RSpec.describe API::Projects do context 'when authenticated' do context 'valid request' do it_behaves_like 'project users response' do - let(:current_user) { user } + let(:project) { project4 } + let(:current_user) { user4 } end end @@ -2011,8 +2006,8 @@ RSpec.describe API::Projects do get api("/projects/#{project.id}/users?skip_users=#{user.id}", user) expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to eq(1) - expect(json_response[0]['id']).to eq(other_user.id) + expect(json_response.size).to eq(2) + expect(json_response.map { |m| m['id'] }).not_to include(user.id) end end end diff --git a/spec/services/static_site_editor/config_service_spec.rb b/spec/services/static_site_editor/config_service_spec.rb new file mode 100644 index 00000000000..5fff4e0af53 --- /dev/null +++ b/spec/services/static_site_editor/config_service_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe StaticSiteEditor::ConfigService do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } + + # params + let(:ref) { double(:ref) } + let(:path) { double(:path) } + let(:return_url) { double(:return_url) } + + # stub data + let(:generated_data) { { generated: true } } + let(:file_data) { { file: true } } + + describe '#execute' do + subject(:execute) do + described_class.new( + container: project, + current_user: user, + params: { + ref: ref, + path: path, + return_url: return_url + } + ).execute + end + + context 'when insufficient permission' do + it 'returns an error' do + expect(execute).to be_error + expect(execute.message).to eq('Insufficient permissions to read configuration') + end + end + + context 'for developer' do + before do + project.add_developer(user) + + allow_next_instance_of(Gitlab::StaticSiteEditor::Config::GeneratedConfig) do |config| + allow(config).to receive(:data) { generated_data } + end + + allow_next_instance_of(Gitlab::StaticSiteEditor::Config::FileConfig) do |config| + allow(config).to receive(:data) { file_data } + end + end + + it 'returns merged generated data and config file data' do + expect(execute).to be_success + expect(execute.payload).to eq(generated: true, file: true) + end + + it 'returns an error if any keys would be overwritten by the merge' do + generated_data[:duplicate_key] = true + file_data[:duplicate_key] = true + expect(execute).to be_error + expect(execute.message).to match(/duplicate key.*duplicate_key.*found/i) + end + end + end +end diff --git a/spec/support/shared_contexts/policies/project_policy_shared_context.rb b/spec/support/shared_contexts/policies/project_policy_shared_context.rb index dd08d81bc46..3016494ac8d 100644 --- a/spec/support/shared_contexts/policies/project_policy_shared_context.rb +++ b/spec/support/shared_contexts/policies/project_policy_shared_context.rb @@ -1,13 +1,17 @@ # frozen_string_literal: true RSpec.shared_context 'ProjectPolicy context' do + let_it_be(:anonymous) { nil } let_it_be(:guest) { create(:user) } let_it_be(:reporter) { create(:user) } let_it_be(:developer) { create(:user) } let_it_be(:maintainer) { create(:user) } let_it_be(:owner) { create(:user) } let_it_be(:admin) { create(:admin) } - let(:project) { create(:project, :public, namespace: owner.namespace) } + let_it_be(:non_member) { create(:user) } + let_it_be_with_refind(:private_project) { create(:project, :private, namespace: owner.namespace) } + let_it_be_with_refind(:internal_project) { create(:project, :internal, namespace: owner.namespace) } + let_it_be_with_refind(:public_project) { create(:project, :public, namespace: owner.namespace) } let(:base_guest_permissions) do %i[ @@ -86,10 +90,12 @@ RSpec.shared_context 'ProjectPolicy context' do let(:maintainer_permissions) { base_maintainer_permissions + additional_maintainer_permissions } let(:owner_permissions) { base_owner_permissions + additional_owner_permissions } - before do - project.add_guest(guest) - project.add_maintainer(maintainer) - project.add_developer(developer) - project.add_reporter(reporter) + before_all do + [private_project, internal_project, public_project].each do |project| + project.add_guest(guest) + project.add_reporter(reporter) + project.add_developer(developer) + project.add_maintainer(maintainer) + end end end diff --git a/spec/support/shared_contexts/requests/api/conan_packages_shared_context.rb b/spec/support/shared_contexts/requests/api/conan_packages_shared_context.rb new file mode 100644 index 00000000000..580ebf00dcb --- /dev/null +++ b/spec/support/shared_contexts/requests/api/conan_packages_shared_context.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +RSpec.shared_context 'conan api setup' do + include PackagesManagerApiSpecHelpers + include HttpBasicAuthHelpers + + let(:package) { create(:conan_package) } + let_it_be(:personal_access_token) { create(:personal_access_token) } + let_it_be(:user) { personal_access_token.user } + let_it_be(:base_secret) { SecureRandom.base64(64) } + let_it_be(:job) { create(:ci_build, :running, user: user) } + let_it_be(:job_token) { job.token } + let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) } + + let(:project) { package.project } + let(:auth_token) { personal_access_token.token } + let(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) } + + let(:headers) do + { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', auth_token) } + end + + let(:jwt_secret) do + OpenSSL::HMAC.hexdigest( + OpenSSL::Digest::SHA256.new, + base_secret, + Gitlab::ConanToken::HMAC_KEY + ) + end + + before do + project.add_developer(user) + allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret) + end +end + +RSpec.shared_context 'conan recipe endpoints' do + include PackagesManagerApiSpecHelpers + include HttpBasicAuthHelpers + + let(:jwt) { build_jwt(personal_access_token) } + let(:headers) { build_token_auth_header(jwt.encoded) } + let(:conan_package_reference) { '123456789' } + let(:presenter) { double('::Packages::Conan::PackagePresenter') } + + before do + allow(::Packages::Conan::PackagePresenter).to receive(:new) + .with(package, user, package.project, any_args) + .and_return(presenter) + end +end + +RSpec.shared_context 'conan file download endpoints' do + include PackagesManagerApiSpecHelpers + include HttpBasicAuthHelpers + + let(:jwt) { build_jwt(personal_access_token) } + let(:headers) { build_token_auth_header(jwt.encoded) } + let(:recipe_path) { package.conan_recipe_path } + let(:package_file) { package.package_files.find_by(file_name: 'conaninfo.txt') } + let(:recipe_file) { package.package_files.find_by(file_name: 'conanfile.py') } + let(:metadata) { package_file.conan_file_metadatum } +end + +RSpec.shared_context 'conan file upload endpoints' do + include PackagesManagerApiSpecHelpers + include WorkhorseHelpers + include HttpBasicAuthHelpers + + let(:jwt) { build_jwt(personal_access_token) } + let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } + let(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } } + let(:headers_with_token) { build_token_auth_header(jwt.encoded).merge(workhorse_header) } + let(:recipe_path) { "foo/bar/#{project.full_path.tr('/', '+')}/baz"} +end diff --git a/spec/support/shared_examples/policies/project_policy_shared_examples.rb b/spec/support/shared_examples/policies/project_policy_shared_examples.rb index d8476f5dcc2..d05e5eb9120 100644 --- a/spec/support/shared_examples/policies/project_policy_shared_examples.rb +++ b/spec/support/shared_examples/policies/project_policy_shared_examples.rb @@ -59,8 +59,7 @@ RSpec.shared_examples 'project policies as anonymous' do let(:project) { create(:project, :public, namespace: group) } let(:user_permissions) { [:create_merge_request_in, :create_project, :create_issue, :create_note, :upload_file, :award_emoji] } let(:anonymous_permissions) { guest_permissions - user_permissions } - - subject { described_class.new(nil, project) } + let(:current_user) { anonymous } before do create(:group_member, :invited, group: group) @@ -78,9 +77,8 @@ RSpec.shared_examples 'project policies as anonymous' do end context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(nil, project) } + let(:project) { private_project } + let(:current_user) { anonymous } it { is_expected.to be_banned } end @@ -109,10 +107,10 @@ RSpec.shared_examples 'deploy token does not get confused with user' do end RSpec.shared_examples 'project policies as guest' do - subject { described_class.new(guest, project) } - context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } + let(:project) { private_project } + let(:current_user) { guest } + let(:reporter_public_build_permissions) do reporter_permissions - [:read_build, :read_pipeline] end @@ -167,9 +165,8 @@ end RSpec.shared_examples 'project policies as reporter' do context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(reporter, project) } + let(:project) { private_project } + let(:current_user) { reporter } it do expect_allowed(*guest_permissions) @@ -192,9 +189,8 @@ end RSpec.shared_examples 'project policies as developer' do context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(developer, project) } + let(:project) { private_project } + let(:current_user) { developer } it do expect_allowed(*guest_permissions) @@ -217,9 +213,8 @@ end RSpec.shared_examples 'project policies as maintainer' do context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(maintainer, project) } + let(:project) { private_project } + let(:current_user) { maintainer } it do expect_allowed(*guest_permissions) @@ -242,9 +237,8 @@ end RSpec.shared_examples 'project policies as owner' do context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(owner, project) } + let(:project) { private_project } + let(:current_user) { owner } it do expect_allowed(*guest_permissions) @@ -267,9 +261,8 @@ end RSpec.shared_examples 'project policies as admin with admin mode' do context 'abilities for non-public projects', :enable_admin_mode do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(admin, project) } + let(:project) { private_project } + let(:current_user) { admin } it do expect_allowed(*guest_permissions) @@ -316,9 +309,8 @@ end RSpec.shared_examples 'project policies as admin without admin mode' do context 'abilities for non-public projects' do - let(:project) { create(:project, namespace: owner.namespace) } - - subject { described_class.new(admin, project) } + let(:project) { private_project } + let(:current_user) { admin } it { is_expected.to be_banned } diff --git a/spec/support/shared_examples/requests/api/conan_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/conan_packages_shared_examples.rb new file mode 100644 index 00000000000..c56290a0aa9 --- /dev/null +++ b/spec/support/shared_examples/requests/api/conan_packages_shared_examples.rb @@ -0,0 +1,843 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'conan ping endpoint' do + it 'responds with 401 Unauthorized when no token provided' do + get api(url) + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + it 'responds with 200 OK when valid token is provided' do + jwt = build_jwt(personal_access_token) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:ok) + expect(response.headers['X-Conan-Server-Capabilities']).to eq("") + end + + it 'responds with 200 OK when valid job token is provided' do + jwt = build_jwt_from_job(job) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:ok) + expect(response.headers['X-Conan-Server-Capabilities']).to eq("") + end + + it 'responds with 200 OK when valid deploy token is provided' do + jwt = build_jwt_from_deploy_token(deploy_token) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:ok) + expect(response.headers['X-Conan-Server-Capabilities']).to eq("") + end + + it 'responds with 401 Unauthorized when invalid access token ID is provided' do + jwt = build_jwt(double(id: 12345), user_id: personal_access_token.user_id) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + it 'responds with 401 Unauthorized when invalid user is provided' do + jwt = build_jwt(personal_access_token, user_id: 12345) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + it 'responds with 401 Unauthorized when the provided JWT is signed with different secret' do + jwt = build_jwt(personal_access_token, secret: SecureRandom.base64(32)) + get api(url), headers: build_token_auth_header(jwt.encoded) + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + it 'responds with 401 Unauthorized when invalid JWT is provided' do + get api(url), headers: build_token_auth_header('invalid-jwt') + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + context 'packages feature disabled' do + it 'responds with 404 Not Found' do + stub_packages_setting(enabled: false) + get api(url) + + expect(response).to have_gitlab_http_status(:not_found) + end + end +end + +RSpec.shared_examples 'conan search endpoint' do + before do + project.update_column(:visibility_level, Gitlab::VisibilityLevel::PUBLIC) + + get api(url), headers: headers, params: params + end + + subject { json_response['results'] } + + context 'returns packages with a matching name' do + let(:params) { { q: package.conan_recipe } } + + it { is_expected.to contain_exactly(package.conan_recipe) } + end + + context 'returns packages using a * wildcard' do + let(:params) { { q: "#{package.name[0, 3]}*" } } + + it { is_expected.to contain_exactly(package.conan_recipe) } + end + + context 'does not return non-matching packages' do + let(:params) { { q: "foo" } } + + it { is_expected.to be_blank } + end +end + +RSpec.shared_examples 'conan authenticate endpoint' do + subject { get api(url), headers: headers } + + context 'when using invalid token' do + let(:auth_token) { 'invalid_token' } + + it 'responds with 401' do + subject + + expect(response).to have_gitlab_http_status(:unauthorized) + end + end + + context 'when valid JWT access token is provided' do + it 'responds with 200' do + subject + + expect(response).to have_gitlab_http_status(:ok) + end + + it 'token has valid validity time' do + freeze_time do + subject + + payload = JSONWebToken::HMACToken.decode( + response.body, jwt_secret).first + expect(payload['access_token']).to eq(personal_access_token.id) + expect(payload['user_id']).to eq(personal_access_token.user_id) + + duration = payload['exp'] - payload['iat'] + expect(duration).to eq(1.hour) + end + end + end + + context 'with valid job token' do + let(:auth_token) { job_token } + + it 'responds with 200' do + subject + + expect(response).to have_gitlab_http_status(:ok) + end + end + + context 'with valid deploy token' do + let(:auth_token) { deploy_token.token } + + it 'responds with 200' do + subject + + expect(response).to have_gitlab_http_status(:ok) + end + end +end + +RSpec.shared_examples 'conan check_credentials endpoint' do + it 'responds with a 200 OK with PAT' do + get api(url), headers: headers + + expect(response).to have_gitlab_http_status(:ok) + end + + context 'with job token' do + let(:auth_token) { job_token } + + it 'responds with a 200 OK with job token' do + get api(url), headers: headers + + expect(response).to have_gitlab_http_status(:ok) + end + end + + context 'with deploy token' do + let(:auth_token) { deploy_token.token } + + it 'responds with a 200 OK with job token' do + get api(url), headers: headers + + expect(response).to have_gitlab_http_status(:ok) + end + end + + it 'responds with a 401 Unauthorized when an invalid token is used' do + get api(url), headers: build_token_auth_header('invalid-token') + + expect(response).to have_gitlab_http_status(:unauthorized) + end +end + +RSpec.shared_examples 'rejects invalid recipe' do + context 'with invalid recipe path' do + let(:recipe_path) { '../../foo++../..' } + + it 'returns 400' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end +end + +RSpec.shared_examples 'rejects invalid file_name' do |invalid_file_name| + let(:file_name) { invalid_file_name } + + context 'with invalid file_name' do + it 'returns 400' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end +end + +RSpec.shared_examples 'rejects recipe for invalid project' do + context 'with invalid project' do + let(:recipe_path) { 'aa/bb/cc/dd' } + let(:project_id) { 9999 } + + it_behaves_like 'not found request' + end +end + +RSpec.shared_examples 'empty recipe for not found package' do + context 'with invalid recipe url' do + let(:recipe_path) do + 'aa/bb/%{project}/ccc' % { project: ::Packages::Conan::Metadatum.package_username_from(full_path: project.full_path) } + end + + it 'returns not found' do + allow(::Packages::Conan::PackagePresenter).to receive(:new) + .with( + nil, + user, + project, + any_args + ).and_return(presenter) + allow(presenter).to receive(:recipe_snapshot) { {} } + allow(presenter).to receive(:package_snapshot) { {} } + + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.body).to eq("{}") + end + end +end + +RSpec.shared_examples 'not selecting a package with the wrong type' do + context 'with a nuget package with same name and version' do + let(:conan_username) { ::Packages::Conan::Metadatum.package_username_from(full_path: project.full_path) } + let(:wrong_package) { create(:nuget_package, name: "wrong", version: '1.0.0', project: project) } + let(:recipe_path) { "#{wrong_package.name}/#{wrong_package.version}/#{conan_username}/foo" } + + it 'calls the presenter with a nil package' do + expect(::Packages::Conan::PackagePresenter).to receive(:new) + .with(nil, user, project, any_args) + + subject + end + end +end + +RSpec.shared_examples 'recipe download_urls' do + let(:recipe_path) { package.conan_recipe_path } + + it 'returns the download_urls for the recipe files' do + expected_response = { + 'conanfile.py' => "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanfile.py", + 'conanmanifest.txt' => "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" + } + + allow(presenter).to receive(:recipe_urls) { expected_response } + + subject + + expect(json_response).to eq(expected_response) + end + + it_behaves_like 'not selecting a package with the wrong type' +end + +RSpec.shared_examples 'package download_urls' do + let(:recipe_path) { package.conan_recipe_path } + + it 'returns the download_urls for the package files' do + expected_response = { + 'conaninfo.txt' => "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt", + 'conanmanifest.txt' => "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conanmanifest.txt", + 'conan_package.tgz' => "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conan_package.tgz" + } + + allow(presenter).to receive(:package_urls) { expected_response } + + subject + + expect(json_response).to eq(expected_response) + end + + it_behaves_like 'not selecting a package with the wrong type' +end + +RSpec.shared_examples 'rejects invalid upload_url params' do + context 'with unaccepted json format' do + let(:params) { %w[foo bar] } + + it 'returns 400' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end +end + +RSpec.shared_examples 'successful response when using Unicorn' do + context 'on Unicorn', :unicorn do + it 'returns successfully' do + subject + + expect(response).to have_gitlab_http_status(:ok) + end + end +end + +RSpec.shared_examples 'recipe snapshot endpoint' do + subject { get api(url), headers: headers } + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects recipe for invalid project' + it_behaves_like 'empty recipe for not found package' + + context 'with existing package' do + it 'returns a hash of files with their md5 hashes' do + expected_response = { + 'conanfile.py' => 'md5hash1', + 'conanmanifest.txt' => 'md5hash2' + } + + allow(presenter).to receive(:recipe_snapshot) { expected_response } + + subject + + expect(json_response).to eq(expected_response) + end + end +end + +RSpec.shared_examples 'package snapshot endpoint' do + subject { get api(url), headers: headers } + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects recipe for invalid project' + it_behaves_like 'empty recipe for not found package' + + context 'with existing package' do + it 'returns a hash of md5 values for the files' do + expected_response = { + 'conaninfo.txt' => "md5hash1", + 'conanmanifest.txt' => "md5hash2", + 'conan_package.tgz' => "md5hash3" + } + + allow(presenter).to receive(:package_snapshot) { expected_response } + + subject + + expect(json_response).to eq(expected_response) + end + end +end + +RSpec.shared_examples 'recipe download_urls endpoint' do + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects recipe for invalid project' + it_behaves_like 'recipe download_urls' +end + +RSpec.shared_examples 'package download_urls endpoint' do + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects recipe for invalid project' + it_behaves_like 'package download_urls' +end + +RSpec.shared_examples 'recipe upload_urls endpoint' do + let(:recipe_path) { package.conan_recipe_path } + + let(:params) do + { 'conanfile.py': 24, + 'conanmanifest.txt': 123 } + end + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects invalid upload_url params' + it_behaves_like 'successful response when using Unicorn' + + it 'returns a set of upload urls for the files requested' do + subject + + expected_response = { + 'conanfile.py': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanfile.py", + 'conanmanifest.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" + } + + expect(response.body).to eq(expected_response.to_json) + end + + context 'with conan_sources and conan_export files' do + let(:params) do + { 'conan_sources.tgz': 345, + 'conan_export.tgz': 234, + 'conanmanifest.txt': 123 } + end + + it 'returns upload urls for the additional files' do + subject + + expected_response = { + 'conan_sources.tgz': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_sources.tgz", + 'conan_export.tgz': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conan_export.tgz", + 'conanmanifest.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" + } + + expect(response.body).to eq(expected_response.to_json) + end + end + + context 'with an invalid file' do + let(:params) do + { 'invalid_file.txt': 10, + 'conanmanifest.txt': 123 } + end + + it 'does not return the invalid file as an upload_url' do + subject + + expected_response = { + 'conanmanifest.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/export/conanmanifest.txt" + } + + expect(response.body).to eq(expected_response.to_json) + end + end +end + +RSpec.shared_examples 'package upload_urls endpoint' do + let(:recipe_path) { package.conan_recipe_path } + + let(:params) do + { 'conaninfo.txt': 24, + 'conanmanifest.txt': 123, + 'conan_package.tgz': 523 } + end + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects invalid upload_url params' + it_behaves_like 'successful response when using Unicorn' + + it 'returns a set of upload urls for the files requested' do + expected_response = { + 'conaninfo.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt", + 'conanmanifest.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conanmanifest.txt", + 'conan_package.tgz': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conan_package.tgz" + } + + subject + + expect(response.body).to eq(expected_response.to_json) + end + + context 'with invalid files' do + let(:params) do + { 'conaninfo.txt': 24, + 'invalid_file.txt': 10 } + end + + it 'returns upload urls only for the valid requested files' do + expected_response = { + 'conaninfo.txt': "#{url_prefix}/packages/conan/v1/files/#{package.conan_recipe_path}/0/package/123456789/0/conaninfo.txt" + } + + subject + + expect(response.body).to eq(expected_response.to_json) + end + end +end + +RSpec.shared_examples 'delete package endpoint' do + let(:recipe_path) { package.conan_recipe_path } + + it_behaves_like 'rejects invalid recipe' + + it 'returns unauthorized for users without valid permission' do + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + + context 'with delete permissions' do + before do + project.add_maintainer(user) + end + + it_behaves_like 'a gitlab tracking event', 'API::ConanPackages', 'delete_package' + + it 'deletes a package' do + expect { subject }.to change { Packages::Package.count }.from(2).to(1) + end + end +end + +RSpec.shared_examples 'denies download with no token' do + context 'with no private token' do + let(:headers) { {} } + + it 'returns 400' do + subject + + expect(response).to have_gitlab_http_status(:unauthorized) + end + end +end + +RSpec.shared_examples 'a public project with packages' do + it 'returns the file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq('application/octet-stream') + end +end + +RSpec.shared_examples 'an internal project with packages' do + before do + project.team.truncate + project.update_column(:visibility_level, Gitlab::VisibilityLevel::INTERNAL) + end + + it_behaves_like 'denies download with no token' + + it 'returns the file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq('application/octet-stream') + end +end + +RSpec.shared_examples 'a private project with packages' do + before do + project.update_column(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + end + + it_behaves_like 'denies download with no token' + + it 'returns the file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq('application/octet-stream') + end + + it 'denies download when not enough permissions' do + project.add_guest(user) + + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end +end + +RSpec.shared_examples 'not found request' do + it 'returns not found' do + subject + + expect(response).to have_gitlab_http_status(:not_found) + end +end + +RSpec.shared_examples 'recipe file download endpoint' do + it_behaves_like 'a public project with packages' + it_behaves_like 'an internal project with packages' + it_behaves_like 'a private project with packages' +end + +RSpec.shared_examples 'package file download endpoint' do + it_behaves_like 'a public project with packages' + it_behaves_like 'an internal project with packages' + it_behaves_like 'a private project with packages' + + context 'tracking the conan_package.tgz download' do + let(:package_file) { package.package_files.find_by(file_name: ::Packages::Conan::FileMetadatum::PACKAGE_BINARY) } + + it_behaves_like 'a gitlab tracking event', 'API::ConanPackages', 'pull_package' + end +end + +RSpec.shared_examples 'project not found by recipe' do + let(:recipe_path) { 'not/package/for/project' } + + it_behaves_like 'not found request' +end + +RSpec.shared_examples 'project not found by project id' do + let(:project_id) { 99999 } + + it_behaves_like 'not found request' +end + +RSpec.shared_examples 'workhorse authorize endpoint' do + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects invalid file_name', 'conanfile.py.git%2fgit-upload-pack' + it_behaves_like 'workhorse authorization' +end + +RSpec.shared_examples 'workhorse recipe file upload endpoint' do + let(:file_name) { 'conanfile.py' } + let(:params) { { file: temp_file(file_name) } } + + subject do + workhorse_finalize( + url, + method: :put, + file_key: :file, + params: params, + headers: headers_with_token, + send_rewritten_field: true + ) + end + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects invalid file_name', 'conanfile.py.git%2fgit-upload-pack' + it_behaves_like 'uploads a package file' +end + +RSpec.shared_examples 'workhorse package file upload endpoint' do + let(:file_name) { 'conaninfo.txt' } + let(:params) { { file: temp_file(file_name) } } + + subject do + workhorse_finalize( + url, + method: :put, + file_key: :file, + params: params, + headers: headers_with_token, + send_rewritten_field: true + ) + end + + it_behaves_like 'rejects invalid recipe' + it_behaves_like 'rejects invalid file_name', 'conaninfo.txttest' + it_behaves_like 'uploads a package file' + + context 'tracking the conan_package.tgz upload' do + let(:file_name) { ::Packages::Conan::FileMetadatum::PACKAGE_BINARY } + + it_behaves_like 'a gitlab tracking event', 'API::ConanPackages', 'push_package' + end +end + +RSpec.shared_examples 'uploads a package file' do + context 'file size above maximum limit' do + before do + params['file.size'] = project.actual_limits.conan_max_file_size + 1 + end + + it 'handles as a local file' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end + + context 'with object storage disabled' do + context 'without a file from workhorse' do + let(:params) { { file: nil } } + + it 'rejects the request' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end + + context 'with a file' do + it_behaves_like 'package workhorse uploads' + end + + context 'without a token' do + it 'rejects request without a token' do + headers_with_token.delete('HTTP_AUTHORIZATION') + + subject + + expect(response).to have_gitlab_http_status(:unauthorized) + end + end + + context 'when params from workhorse are correct' do + it 'creates package and stores package file' do + expect { subject } + .to change { project.packages.count }.by(1) + .and change { Packages::PackageFile.count }.by(1) + + expect(response).to have_gitlab_http_status(:ok) + + package_file = project.packages.last.package_files.reload.last + expect(package_file.file_name).to eq(params[:file].original_filename) + end + + it "doesn't attempt to migrate file to object storage" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + end + + context 'with object storage enabled' do + context 'and direct upload enabled' do + let!(:fog_connection) do + stub_package_file_object_storage(direct_upload: true) + end + + let(:tmp_object) do + fog_connection.directories.new(key: 'packages').files.create( # rubocop:disable Rails/SaveBang + key: "tmp/uploads/#{file_name}", + body: 'content' + ) + end + + let(:fog_file) { fog_to_uploaded_file(tmp_object) } + + ['123123', '../../123123'].each do |remote_id| + context "with invalid remote_id: #{remote_id}" do + let(:params) do + { + file: fog_file, + 'file.remote_id' => remote_id + } + end + + it 'responds with status 403' do + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + end + + context 'with valid remote_id' do + let(:params) do + { + file: fog_file, + 'file.remote_id' => file_name + } + end + + it 'creates package and stores package file' do + expect { subject } + .to change { project.packages.count }.by(1) + .and change { Packages::PackageFile.count }.by(1) + + expect(response).to have_gitlab_http_status(:ok) + + package_file = project.packages.last.package_files.reload.last + expect(package_file.file_name).to eq(params[:file].original_filename) + expect(package_file.file.read).to eq('content') + end + end + end + + it_behaves_like 'background upload schedules a file migration' + end +end + +RSpec.shared_examples 'workhorse authorization' do + it 'authorizes posting package with a valid token' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + end + + it 'rejects request without a valid token' do + headers_with_token['HTTP_AUTHORIZATION'] = 'foo' + + subject + + expect(response).to have_gitlab_http_status(:unauthorized) + end + + it 'rejects request without a valid permission' do + project.add_guest(user) + + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + + it 'rejects requests that bypassed gitlab-workhorse' do + headers_with_token.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) + + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + + context 'when using remote storage' do + context 'when direct upload is enabled' do + before do + stub_package_file_object_storage(enabled: true, direct_upload: true) + end + + it 'responds with status 200, location of package remote store and object details' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response).not_to have_key('TempPath') + expect(json_response['RemoteObject']).to have_key('ID') + expect(json_response['RemoteObject']).to have_key('GetURL') + expect(json_response['RemoteObject']).to have_key('StoreURL') + expect(json_response['RemoteObject']).to have_key('DeleteURL') + expect(json_response['RemoteObject']).not_to have_key('MultipartUpload') + end + end + + context 'when direct upload is disabled' do + before do + stub_package_file_object_storage(enabled: true, direct_upload: false) + end + + it 'handles as a local file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).to eq(::Packages::PackageFileUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to be_nil + end + end + end +end diff --git a/spec/support/shared_examples/requests/api/custom_attributes_shared_examples.rb b/spec/support/shared_examples/requests/api/custom_attributes_shared_examples.rb index 0dfc49f2d41..f31cbcfdec1 100644 --- a/spec/support/shared_examples/requests/api/custom_attributes_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/custom_attributes_shared_examples.rb @@ -14,8 +14,7 @@ RSpec.shared_examples 'custom attributes endpoints' do |attributable_name| get api("/#{attributable_name}", user), params: { custom_attributes: { foo: 'foo', bar: 'bar' } } expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to be 2 - expect(json_response.map { |r| r['id'] }).to contain_exactly attributable.id, other_attributable.id + expect(json_response.map { |r| r['id'] }).to include(attributable.id, other_attributable.id) end end @@ -40,7 +39,7 @@ RSpec.shared_examples 'custom attributes endpoints' do |attributable_name| get api("/#{attributable_name}", user), params: { with_custom_attributes: true } expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to be 2 + expect(json_response).not_to be_empty expect(json_response.first).not_to include 'custom_attributes' end end @@ -50,16 +49,15 @@ RSpec.shared_examples 'custom attributes endpoints' do |attributable_name| get api("/#{attributable_name}", admin) expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to be 2 + expect(json_response).not_to be_empty expect(json_response.first).not_to include 'custom_attributes' - expect(json_response.second).not_to include 'custom_attributes' end it 'includes custom attributes if requested' do get api("/#{attributable_name}", admin), params: { with_custom_attributes: true } expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to be 2 + expect(json_response).not_to be_empty attributable_response = json_response.find { |r| r['id'] == attributable.id } other_attributable_response = json_response.find { |r| r['id'] == other_attributable.id } diff --git a/yarn.lock b/yarn.lock index 4782fed4382..1b86922873e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -848,10 +848,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.164.0.tgz#6cefad871c45f945ef92b99015d0f510b1d2de4a" integrity sha512-a9e/cYUc1QQk7azjH4x/m6/p3icavwGEi5F9ipNlDqiJtUor5tqojxvMxPOhuVbN/mTwnC6lGsSZg4tqTsdJAQ== -"@gitlab/ui@21.2.1": - version "21.2.1" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-21.2.1.tgz#260e308e82374224095aea59a891f8a458eace3e" - integrity sha512-9ne5kPMGCVHSSeWmMzcnXgB4JPKhbVQ520FvM+z3MkvTaQrxhh4H/wKD4durBusTokbrRXsOhqtuV/NI1WJjDQ== +"@gitlab/ui@21.3.0": + version "21.3.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-21.3.0.tgz#1cdcfed11e9aa730af1b1a9102390cab81c36928" + integrity sha512-6k02S4y1wNnxmPpHOe5WA0FIeAfzs2M06Xz1CCYQ+7M7P0PBaG0FFW+uvWyVnnp5CUaL2frMBn9TXD9pXdOeKg== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.3.0"