From 265a7ceccadf01cf1c2983c54abf86de19f6c2ad Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 21 Jul 2022 09:09:01 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/services/issuable/clone/base_service.rb | 1 + app/services/issues/clone_service.rb | 11 +++-- app/services/system_note_service.rb | 4 +- .../system_notes/issuables_service.rb | 5 ++- ...al_cost_factor_for_gitlab_contributors.yml | 8 ---- doc/api/oauth2.md | 9 ++-- doc/ci/pipelines/cicd_minutes.md | 24 +++++++++++ doc/integration/azure.md | 43 ++++++++++++++++--- doc/integration/gitlab.md | 8 +++- lib/api/repositories.rb | 10 ++++- lib/gitlab/pagination/gitaly_keyset_pager.rb | 8 +++- locale/gitlab.pot | 9 ++++ .../pagination/gitaly_keyset_pager_spec.rb | 14 ++++++ spec/requests/api/repositories_spec.rb | 26 +++++++++++ spec/services/issues/clone_service_spec.rb | 16 ++++++- spec/services/issues/move_service_spec.rb | 9 ++-- spec/services/system_note_service_spec.rb | 5 ++- .../system_notes/issuables_service_spec.rb | 20 ++++++++- workhorse/go.mod | 2 +- workhorse/go.sum | 4 +- 20 files changed, 194 insertions(+), 42 deletions(-) delete mode 100644 config/feature_flags/development/ci_minimal_cost_factor_for_gitlab_contributors.yml diff --git a/app/services/issuable/clone/base_service.rb b/app/services/issuable/clone/base_service.rb index 98c50347719..3c13944cfbc 100644 --- a/app/services/issuable/clone/base_service.rb +++ b/app/services/issuable/clone/base_service.rb @@ -16,6 +16,7 @@ module Issuable # ApplicationRecord.transaction do @new_entity = create_new_entity + @new_entity.system_note_timestamp = nil update_new_entity update_old_entity diff --git a/app/services/issues/clone_service.rb b/app/services/issues/clone_service.rb index 896b15a14b8..d054cf7827d 100644 --- a/app/services/issues/clone_service.rb +++ b/app/services/issues/clone_service.rb @@ -96,9 +96,14 @@ module Issues end def add_note_from - SystemNoteService.noteable_cloned(new_entity, target_project, - original_entity, current_user, - direction: :from) + SystemNoteService.noteable_cloned( + new_entity, + target_project, + original_entity, + current_user, + direction: :from, + created_at: new_entity.created_at + ) end def add_note_to diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index 0a3a0ca00b5..fb24ace9fdb 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -256,8 +256,8 @@ module SystemNoteService ::SystemNotes::IssuablesService.new(noteable: noteable, project: project, author: author).noteable_moved(noteable_ref, direction) end - def noteable_cloned(noteable, project, noteable_ref, author, direction:) - ::SystemNotes::IssuablesService.new(noteable: noteable, project: project, author: author).noteable_cloned(noteable_ref, direction) + def noteable_cloned(noteable, project, noteable_ref, author, direction:, created_at: nil) + ::SystemNotes::IssuablesService.new(noteable: noteable, project: project, author: author).noteable_cloned(noteable_ref, direction, created_at: created_at) end def mark_duplicate_issue(noteable, project, author, canonical_issue) diff --git a/app/services/system_notes/issuables_service.rb b/app/services/system_notes/issuables_service.rb index 8da316987b1..440ffd78855 100644 --- a/app/services/system_notes/issuables_service.rb +++ b/app/services/system_notes/issuables_service.rb @@ -312,13 +312,14 @@ module SystemNotes # # noteable_ref - Referenced noteable # direction - symbol, :to or :from + # created_at - timestamp for the system note, defaults to current time # # Example Note text: # # "cloned to some_namespace/project_new#11" # # Returns the created Note object - def noteable_cloned(noteable_ref, direction) + def noteable_cloned(noteable_ref, direction, created_at: nil) unless [:to, :from].include?(direction) raise ArgumentError, "Invalid direction `#{direction}`" end @@ -328,7 +329,7 @@ module SystemNotes issue_activity_counter.track_issue_cloned_action(author: author) if noteable.is_a?(Issue) && direction == :to - create_note(NoteSummary.new(noteable, project, author, body, action: 'cloned')) + create_note(NoteSummary.new(noteable, project, author, body, action: 'cloned', created_at: created_at)) end # Called when the confidentiality changes diff --git a/config/feature_flags/development/ci_minimal_cost_factor_for_gitlab_contributors.yml b/config/feature_flags/development/ci_minimal_cost_factor_for_gitlab_contributors.yml deleted file mode 100644 index e571abdc97f..00000000000 --- a/config/feature_flags/development/ci_minimal_cost_factor_for_gitlab_contributors.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_minimal_cost_factor_for_gitlab_contributors -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89742 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/365862 -milestone: '15.2' -type: development -group: group::pipeline execution -default_enabled: false diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 35c6eb4a982..ab33deda687 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -26,9 +26,12 @@ support [CORS preflight requests](https://developer.mozilla.org/en-US/docs/Web/H - `/oauth/token` - `/oauth/userinfo` -In addition to the headers listed for [simple requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS#simple_requests), -only the `Authorization` header can be used for preflight requests. For example, the `X-Requested-With` header -can't be used for preflight requests. +Only certain headers can be used for preflight requests: + +- The headers listed for [simple requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS#simple_requests). +- The `Authorization` header. + +For example, the `X-Requested-With` header can't be used for preflight requests. ## Supported OAuth 2.0 flows diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index 1e862c87035..e03d926ec93 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -204,12 +204,36 @@ The cost factors for jobs running on shared runners on GitLab.com are: - `0.008` for public projects, and projects in the [GitLab for Open Source program](../../subscriptions/index.md#gitlab-for-open-source). For every 125 minutes of job execution time, you use 1 CI/CD minute. - `1` for internal and private projects. +- Calculated differently for [community contributions to GitLab projects](#cost-factor-for-community-contributions-to-gitlab-projects). The cost factors on self-managed instances are: - `0` for public projects, so they do not consume CI/CD minutes. - `1` for internal and private projects. +#### Cost factor for community contributions to GitLab projects + +Community contributors can use up to 300,000 minutes on shared runners when +contributing to open source projects maintained by GitLab. The 300,000 +minutes applies to all SaaS tiers, and the cost factor calculation is: + +- `Monthly minute quota / 300,000 job duration minutes = Cost factor` + +For example, with the 10,000 CI/CD minutes per month in the Premium tier: + +- 10,000 / 300,000 = 0.03333333333 cost factor. + +For this reduced cost factor: + +- The merge request source project must be a fork of a GitLab-maintained project, + such as [`gitlab-com/www-gitlab-com`](https://gitlab.com/gitlab-com/www-gitlab-com), + [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab), and so on. +- The merge request target project must be the fork's parent project. +- The pipeline must be a merge request, merged results, or merge train pipeline. + +GitLab administrators can add a namespace to the reduced cost factor +[with a flag](../../administration/feature_flags.md) named `ci_minimal_cost_factor_for_gitlab_namespaces`. + ### Additional costs on GitLab SaaS GitLab SaaS shared runners have different cost factors, depending on the runner type (Linux, Windows, macOS) and the virtual machine configuration. diff --git a/doc/integration/azure.md b/doc/integration/azure.md index 515e7406545..da1aa574bd6 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -107,6 +107,24 @@ Alternatively, add the `User.Read.All` application permission. ] ``` + For [alternative Azure clouds](https://docs.microsoft.com/en-us/azure/active-directory/develop/authentication-national-cloud), + configure `base_azure_url` under the `args` section. For example, for Azure Government Community Cloud (GCC): + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "azure_activedirectory_v2", + "label" => "Provider name", # optional label for login button, defaults to "Azure AD v2" + "args" => { + "client_id" => "CLIENT ID", + "client_secret" => "CLIENT SECRET", + "tenant_id" => "TENANT ID", + "base_azure_url" => "https://login.microsoftonline.us" + } + } + ] + ``` + - **For installations from source** For the v1.0 endpoint: @@ -115,8 +133,8 @@ Alternatively, add the `User.Read.All` application permission. - { name: 'azure_oauth2', # label: 'Provider name', # optional label for login button, defaults to "Azure AD" args: { client_id: 'CLIENT ID', - client_secret: 'CLIENT SECRET', - tenant_id: 'TENANT ID' } } + client_secret: 'CLIENT SECRET', + tenant_id: 'TENANT ID' } } ``` For the v2.0 endpoint: @@ -125,14 +143,25 @@ Alternatively, add the `User.Read.All` application permission. - { name: 'azure_activedirectory_v2', label: 'Provider name', # optional label for login button, defaults to "Azure AD v2" args: { client_id: "CLIENT ID", - client_secret: "CLIENT SECRET", - tenant_id: "TENANT ID" } } + client_secret: "CLIENT SECRET", + tenant_id: "TENANT ID" } } ``` - You can optionally add the following parameters: + For [alternative Azure clouds](https://docs.microsoft.com/en-us/azure/active-directory/develop/authentication-national-cloud), + configure `base_azure_url` under the `args` section. For example, for Azure Government Community Cloud (GCC): - - `base_azure_url` for different locales. For example, `base_azure_url: "https://login.microsoftonline.de"`. - - `scope`, which you add to `args`. The default is `openid profile email`. + ```yaml + - { name: 'azure_activedirectory_v2', + label: 'Provider name', # optional label for login button, defaults to "Azure AD v2" + args: { client_id: "CLIENT ID", + client_secret: "CLIENT SECRET", + tenant_id: "TENANT ID", + base_azure_url: "https://login.microsoftonline.us" } } + ``` + + In addition, you can optionally add the following parameters to the `args` section: + + - `scope` for [OAuth2 scopes](https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-auth-code-flow). The default is `openid profile email`. 1. Save the configuration file. diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 02705d9dec3..fee1e573384 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -77,7 +77,7 @@ GitLab.com generates an application ID and secret key for you to use. app_id: "YOUR_APP_ID", app_secret: "YOUR_APP_SECRET", args: { scope: "read_user" # optional: defaults to the scopes of the application - , client_options: { site: "https://gitlab.example.com/api/v4" } } + , client_options: { site: "https://gitlab.example.com" } } } ] ``` @@ -98,9 +98,13 @@ GitLab.com generates an application ID and secret key for you to use. label: 'Provider name', # optional label for login button, defaults to "GitLab.com" app_id: 'YOUR_APP_ID', app_secret: 'YOUR_APP_SECRET', - args: { "client_options": { "site": 'https://gitlab.example.com/api/v4' } } + args: { "client_options": { "site": 'https://gitlab.example.com' } } ``` + NOTE: + In GitLab 15.1 and earlier, the `site` parameter requires an `/api/v4` suffix. + We recommend you drop this suffix after you upgrade to GitLab 15.2 or later. + 1. Change `'YOUR_APP_ID'` to the Application ID from the GitLab.com application page. 1. Change `'YOUR_APP_SECRET'` to the secret from the GitLab.com application page. 1. Save the configuration file. diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4c7cc6be8b6..cef72d898e6 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -99,11 +99,17 @@ module API optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' use :pagination - optional :pagination, type: String, values: %w(legacy keyset), default: 'legacy', desc: 'Specify the pagination method' + optional :pagination, type: String, values: %w(legacy keyset none), default: 'legacy', desc: 'Specify the pagination method ("none" is only valid if "recursive" is true)' - given pagination: -> (value) { value == 'keyset' } do + given pagination: ->(value) { value == 'keyset' } do optional :page_token, type: String, desc: 'Record from which to start the keyset pagination' end + + given pagination: ->(value) { value == 'none' } do + given recursive: ->(value) { value == false } do + validates([:pagination], except_values: { value: 'none', message: 'cannot be "none" unless "recursive" is true' }) + end + end end get ':id/repository/tree', urgency: :low do tree_finder = ::Repositories::TreeFinder.new(user_project, declared_params(include_missing: false)) diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 8bbc9a93610..1f1061fe4f1 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -12,9 +12,11 @@ module Gitlab @project = project end - # It is expected that the given finder will respond to `execute` method with `gitaly_pagination: true` option + # It is expected that the given finder will respond to `execute` method with `gitaly_pagination:` option # and supports pagination via gitaly. def paginate(finder) + return finder.execute(gitaly_pagination: false) if no_pagination? + return paginate_via_gitaly(finder) if keyset_pagination_enabled?(finder) return paginate_first_page_via_gitaly(finder) if paginate_first_page?(finder) @@ -26,6 +28,10 @@ module Gitlab private + def no_pagination? + params[:pagination] == 'none' + end + def keyset_pagination_enabled?(finder) return false unless params[:pagination] == "keyset" diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 1e5e891074b..59be1e39c5b 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -35099,6 +35099,15 @@ msgstr "" msgid "SecurityReports|Operational vulnerabilities" msgstr "" +msgid "SecurityReports|Parsing errors and warnings in pipeline" +msgstr "" + +msgid "SecurityReports|Parsing errors in pipeline" +msgstr "" + +msgid "SecurityReports|Parsing warnings in pipeline" +msgstr "" + msgid "SecurityReports|Project" msgstr "" diff --git a/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb b/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb index dcb8138bdde..0bafd436bd0 100644 --- a/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb +++ b/spec/lib/gitlab/pagination/gitaly_keyset_pager_spec.rb @@ -126,5 +126,19 @@ RSpec.describe Gitlab::Pagination::GitalyKeysetPager do end end end + + context 'with "none" pagination option' do + let(:expected_result) { double(:result) } + let(:query) { { pagination: 'none' } } + + it 'uses offset pagination' do + expect(finder).to receive(:execute).with(gitaly_pagination: false).and_return(expected_result) + expect(Kaminari).not_to receive(:paginate_array) + expect(Gitlab::Pagination::OffsetPagination).not_to receive(:new) + + actual_result = pager.paginate(finder) + expect(actual_result).to eq(expected_result) + end + end end end diff --git a/spec/requests/api/repositories_spec.rb b/spec/requests/api/repositories_spec.rb index cf0165d123f..3c22f918af5 100644 --- a/spec/requests/api/repositories_spec.rb +++ b/spec/requests/api/repositories_spec.rb @@ -92,6 +92,32 @@ RSpec.describe API::Repositories do expect(json_response.map { |t| t["id"] }).not_to include(page_token) end end + + context 'with pagination=none' do + context 'with recursive=1' do + it 'returns unpaginated recursive project paths tree' do + get api("#{route}?recursive=1&pagination=none", current_user) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response).to be_an Array + expect(response).not_to include_pagination_headers + expect(json_response[4]['name']).to eq('html') + expect(json_response[4]['path']).to eq('files/html') + expect(json_response[4]['type']).to eq('tree') + expect(json_response[4]['mode']).to eq('040000') + end + end + + context 'with recursive=0' do + it 'returns 400' do + get api("#{route}?recursive=0&pagination=none", current_user) + + expect(response).to have_gitlab_http_status(:bad_request) + expect(json_response['error']) + .to eq('pagination cannot be "none" unless "recursive" is true') + end + end + end end context 'when unauthenticated', 'and project is public' do diff --git a/spec/services/issues/clone_service_spec.rb b/spec/services/issues/clone_service_spec.rb index 858dfc4ab3a..c967f9ad1f9 100644 --- a/spec/services/issues/clone_service_spec.rb +++ b/spec/services/issues/clone_service_spec.rb @@ -57,8 +57,20 @@ RSpec.describe Issues::CloneService do expect(old_issue.notes.last.note).to start_with 'cloned to' end - it 'adds system note to new issue at the end' do - expect(new_issue.notes.last.note).to start_with 'cloned from' + it 'adds system note to new issue at the start' do + # We set an assignee so an assignee system note will be generated and + # we can assert that the "cloned from" note is the first one + assignee = create(:user) + new_project.add_developer(assignee) + old_issue.assignees = [assignee] + + new_issue = clone_service.execute(old_issue, new_project) + + expect(new_issue.notes.size).to eq(2) + + cloned_from_note = new_issue.notes.last + expect(cloned_from_note.note).to start_with 'cloned from' + expect(new_issue.notes.fresh.first).to eq(cloned_from_note) end it 'keeps old issue open' do diff --git a/spec/services/issues/move_service_spec.rb b/spec/services/issues/move_service_spec.rb index 5a1bb2e8b74..c130863f928 100644 --- a/spec/services/issues/move_service_spec.rb +++ b/spec/services/issues/move_service_spec.rb @@ -16,7 +16,7 @@ RSpec.describe Issues::MoveService do let_it_be(:new_project) { create(:project, namespace: sub_group_2) } let(:old_issue) do - create(:issue, title: title, description: description, project: old_project, author: author) + create(:issue, title: title, description: description, project: old_project, author: author, created_at: 1.day.ago, updated_at: 1.day.ago) end subject(:move_service) do @@ -62,8 +62,11 @@ RSpec.describe Issues::MoveService do expect(old_issue.notes.last.note).to start_with 'moved to' end - it 'adds system note to new issue at the end' do - expect(new_issue.notes.last.note).to start_with 'moved from' + it 'adds system note to new issue at the end', :freeze_time do + system_note = new_issue.notes.last + + expect(system_note.note).to start_with 'moved from' + expect(system_note.created_at).to be_like_time(Time.current) end it 'closes old issue' do diff --git a/spec/services/system_note_service_spec.rb b/spec/services/system_note_service_spec.rb index 6f15a53da17..aa3829d9ea5 100644 --- a/spec/services/system_note_service_spec.rb +++ b/spec/services/system_note_service_spec.rb @@ -375,13 +375,14 @@ RSpec.describe SystemNoteService do describe '.noteable_cloned' do let(:noteable_ref) { double } let(:direction) { double } + let(:created_at) { double } it 'calls IssuableService' do expect_next_instance_of(::SystemNotes::IssuablesService) do |service| - expect(service).to receive(:noteable_cloned).with(noteable_ref, direction) + expect(service).to receive(:noteable_cloned).with(noteable_ref, direction, created_at: created_at) end - described_class.noteable_cloned(double, double, noteable_ref, double, direction: direction) + described_class.noteable_cloned(double, double, noteable_ref, double, direction: direction, created_at: created_at) end end diff --git a/spec/services/system_notes/issuables_service_spec.rb b/spec/services/system_notes/issuables_service_spec.rb index 41e4ab1c040..4bfedf5c2a2 100644 --- a/spec/services/system_notes/issuables_service_spec.rb +++ b/spec/services/system_notes/issuables_service_spec.rb @@ -625,8 +625,8 @@ RSpec.describe ::SystemNotes::IssuablesService do end describe '#noteable_cloned' do - let(:new_project) { create(:project) } - let(:new_noteable) { create(:issue, project: new_project) } + let_it_be(:new_project) { create(:project) } + let_it_be(:new_noteable) { create(:issue, project: new_project) } subject do service.noteable_cloned(new_noteable, direction) @@ -684,6 +684,22 @@ RSpec.describe ::SystemNotes::IssuablesService do end end + context 'custom created timestamp' do + let(:direction) { :from } + + it 'allows setting of custom created_at value' do + timestamp = 1.day.ago + + note = service.noteable_cloned(new_noteable, direction, created_at: timestamp) + + expect(note.created_at).to be_like_time(timestamp) + end + + it 'defaults to current time when created_at is not given', :freeze_time do + expect(subject.created_at).to be_like_time(Time.current) + end + end + context 'metrics' do context 'cloned from' do let(:direction) { :from } diff --git a/workhorse/go.mod b/workhorse/go.mod index 10a0b13b50e..7fb6eb21929 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/Azure/azure-storage-blob-go v0.14.0 - github.com/BurntSushi/toml v1.1.0 + github.com/BurntSushi/toml v1.2.0 github.com/FZambia/sentinel v1.1.0 github.com/alecthomas/chroma/v2 v2.2.0 github.com/aws/aws-sdk-go v1.43.31 diff --git a/workhorse/go.sum b/workhorse/go.sum index ec1ee3eac11..53b3e143f7a 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -145,8 +145,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= +github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=