From 22e9f240efba67d752e33ebdb8ba8205f187dc83 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 16 Jun 2021 09:10:10 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- GITALY_SERVER_VERSION | 2 +- app/services/ci/create_pipeline_service.rb | 1 + app/services/ci/register_job_service.rb | 16 ++--- .../user_project_access_changed_service.rb | 2 +- app/views/projects/services/_form.html.haml | 2 +- app/views/users/show.html.haml | 2 +- app/workers/all_queues.yml | 9 +++ .../user_refresh_from_replica_worker.rb | 15 +++++ .../ci_register_job_service_one_by_one.yml | 8 --- doc/user/infrastructure/index.md | 9 ++- .../pipeline/chain/validate/after_config.rb | 24 +++++++ .../ci/templates/Terraform.gitlab-ci.yml | 63 ++++--------------- locale/gitlab.pot | 30 --------- .../lib/gitlab/ci/templates/templates_spec.rb | 3 +- spec/services/ci/register_job_service_spec.rb | 14 +---- spec/services/projects/create_service_spec.rb | 2 +- .../group_links/create_service_spec.rb | 2 +- .../group_links/destroy_service_spec.rb | 4 +- ...ser_project_access_changed_service_spec.rb | 2 +- .../projects/services/_form.haml_spec.rb | 2 +- .../user_refresh_from_replica_worker_spec.rb | 11 ++++ spec/workers/every_sidekiq_worker_spec.rb | 1 + 22 files changed, 94 insertions(+), 130 deletions(-) create mode 100644 app/workers/authorized_project_update/user_refresh_from_replica_worker.rb delete mode 100644 config/feature_flags/development/ci_register_job_service_one_by_one.yml create mode 100644 lib/gitlab/ci/pipeline/chain/validate/after_config.rb create mode 100644 spec/workers/authorized_project_update/user_refresh_from_replica_worker_spec.rb diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 8bc2e1d2e8a..9b197696144 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -86d069ca736dfde9fa61f8476e974c7b8c15a1e9 +8a6d0e26de9d584941267d2b68c94b37bc30e092 diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index fd333e24860..c039f31aafc 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -13,6 +13,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy, Gitlab::Ci::Pipeline::Chain::Config::Content, Gitlab::Ci::Pipeline::Chain::Config::Process, + Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig, Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs, Gitlab::Ci::Pipeline::Chain::Skip, Gitlab::Ci::Pipeline::Chain::SeedBlock, diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 124912fee77..6280bf4c986 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -125,20 +125,12 @@ module Ci builds = builds.queued_before(params[:job_age].seconds.ago) end - if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true) - build_ids = retrieve_queue(-> { builds.pluck(:id) }) + build_ids = retrieve_queue(-> { builds.pluck(:id) }) - @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) + @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) - build_ids.each do |build_id| - yield Ci::Build.find(build_id) - end - else - builds_array = retrieve_queue(-> { builds.to_a }) - - @metrics.observe_queue_size(-> { builds_array.size }, @runner.runner_type) - - builds_array.each(&blk) + build_ids.each do |build_id| + yield Ci::Build.find(build_id) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/user_project_access_changed_service.rb b/app/services/user_project_access_changed_service.rb index a3e7c4ce5bb..f52502e0379 100644 --- a/app/services/user_project_access_changed_service.rb +++ b/app/services/user_project_access_changed_service.rb @@ -20,7 +20,7 @@ class UserProjectAccessChangedService if priority == HIGH_PRIORITY AuthorizedProjectsWorker.bulk_perform_async(bulk_args) # rubocop:disable Scalability/BulkPerformWithContext else - AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker.bulk_perform_in( # rubocop:disable Scalability/BulkPerformWithContext + AuthorizedProjectUpdate::UserRefreshFromReplicaWorker.bulk_perform_in( # rubocop:disable Scalability/BulkPerformWithContext DELAY, bulk_args, batch_size: 100, batch_delay: 30.seconds) end end diff --git a/app/views/projects/services/_form.html.haml b/app/views/projects/services/_form.html.haml index e8f3f6575aa..724684c9a0a 100644 --- a/app/views/projects/services/_form.html.haml +++ b/app/views/projects/services/_form.html.haml @@ -8,7 +8,7 @@ = form_for(integration, as: :service, url: scoped_integration_path(integration), method: :put, html: { class: 'gl-show-field-errors integration-settings-form js-integration-settings-form', data: { 'test-url' => test_project_service_path(@project, integration) } }) do |form| = render 'shared/service_settings', form: form, integration: integration - %input{ id: 'services_redirect_to', type: 'hidden', name: 'redirect_to', value: request.referrer } + %input{ id: 'services_redirect_to', type: 'hidden', name: 'redirect_to', value: request.referer } - if lookup_context.template_exists?('show', "projects/services/#{integration.to_param}", true) %hr diff --git a/app/views/users/show.html.haml b/app/views/users/show.html.haml index 789854b56a4..effd58ad200 100644 --- a/app/views/users/show.html.haml +++ b/app/views/users/show.html.haml @@ -23,7 +23,7 @@ data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } }> = sprite_icon('error') - else - = link_to new_abuse_report_path(user_id: @user.id, ref_url: request.referrer), class: link_classes + 'btn gl-button btn-default btn-icon', + = link_to new_abuse_report_path(user_id: @user.id, ref_url: request.referer), class: link_classes + 'btn gl-button btn-default btn-icon', title: s_('UserProfile|Report abuse'), data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } do = sprite_icon('error') - if can?(current_user, :read_user_profile, @user) diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 4c68c62f404..31c590183d1 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -30,6 +30,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: authorized_project_update:authorized_project_update_user_refresh_from_replica + :worker_name: AuthorizedProjectUpdate::UserRefreshFromReplicaWorker + :feature_category: :authentication_and_authorization + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: authorized_project_update:authorized_project_update_user_refresh_over_user_range :worker_name: AuthorizedProjectUpdate::UserRefreshOverUserRangeWorker :feature_category: :authentication_and_authorization diff --git a/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb new file mode 100644 index 00000000000..5ca9de63fd7 --- /dev/null +++ b/app/workers/authorized_project_update/user_refresh_from_replica_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module AuthorizedProjectUpdate + class UserRefreshFromReplicaWorker < ::AuthorizedProjectsWorker + feature_category :authentication_and_authorization + urgency :low + queue_namespace :authorized_project_update + deduplicate :until_executing, including_scheduled: true + + idempotent! + + # This worker will start reading data from the replica database soon + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/333219 + end +end diff --git a/config/feature_flags/development/ci_register_job_service_one_by_one.yml b/config/feature_flags/development/ci_register_job_service_one_by_one.yml deleted file mode 100644 index 8f691a01605..00000000000 --- a/config/feature_flags/development/ci_register_job_service_one_by_one.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_register_job_service_one_by_one -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55194 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/323177 -milestone: '13.10' -type: development -group: group::memory -default_enabled: true diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index 0dc8c58c8f6..05ffab93f85 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -20,7 +20,7 @@ for GitLab versions 13.5 and later: ```yaml include: - - template: Terraform.latest.gitlab-ci.yml + - template: Terraform.gitlab-ci.yml variables: # If not using GitLab's HTTP backend, remove this line and specify TF_HTTP_* variables @@ -30,15 +30,14 @@ variables: # TF_ROOT: terraform/production ``` -This template uses `.latest.`, instead of stable, and may include breaking changes. -This template also includes some opinionated decisions, which you can override: +This template includes some opinionated decisions, which you can override: - Including the latest [GitLab Terraform Image](https://gitlab.com/gitlab-org/terraform-images). - Using the [GitLab managed Terraform State](#gitlab-managed-terraform-state) as the Terraform state storage backend. -- Creating [four pipeline stages](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml): +- Creating [four pipeline stages](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml): `init`, `validate`, `build`, and `deploy`. These stages - [run the Terraform commands](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml) + [run the Terraform commands](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml) `init`, `validate`, `plan`, `plan-json`, and `apply`. The `apply` command only runs on `master`. This video from January 2021 walks you through all the GitLab Terraform integration features: diff --git a/lib/gitlab/ci/pipeline/chain/validate/after_config.rb b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb new file mode 100644 index 00000000000..c3db00b4fb2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class AfterConfig < Chain::Base + include Chain::Helpers + + def perform! + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig') diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 6b9db1c2e0f..62b32d7c2db 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,61 +1,22 @@ -# Official image for Hashicorp's Terraform. It uses light image which is Alpine -# based as it is much lighter. -# -# Entrypoint is also needed as image by default set `terraform` binary as an -# entrypoint. -image: - name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - -# Default output file for Terraform plan -variables: - PLAN: plan.tfplan - JSON_PLAN_FILE: tfplan.json - -cache: - paths: - - .terraform - - .terraform.lock.hcl - -before_script: - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - - terraform --version - - terraform init +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: + - init - validate - build - - test - deploy +init: + extends: .init + validate: - stage: validate - script: - - terraform validate + extends: .validate -plan: - stage: build - script: - - terraform plan -out=$PLAN - - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" - artifacts: - paths: - - $PLAN - reports: - terraform: $JSON_PLAN_FILE +build: + extends: .build -# Separate apply job for manual launching Terraform as it can be destructive -# action. -apply: - stage: deploy - environment: - name: production - script: - - terraform apply -input=false $PLAN +deploy: + extends: .deploy dependencies: - - plan - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: manual + - build diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5f00bb98a7a..d0d24dfeb2e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4142,15 +4142,9 @@ msgstr "" msgid "ApprovalRule|Rule name" msgstr "" -msgid "ApprovalRule|Status check" -msgstr "" - msgid "ApprovalRule|Target branch" msgstr "" -msgid "ApprovalRule|Users or groups" -msgstr "" - msgid "ApprovalStatusTooltip|Adheres to separation of duties" msgstr "" @@ -13431,9 +13425,6 @@ msgstr "" msgid "External storage authentication token" msgstr "" -msgid "External url has already been taken" -msgstr "" - msgid "ExternalAuthorizationService|Classification label" msgstr "" @@ -31122,9 +31113,6 @@ msgstr "" msgid "StatusCheck|External API is already in use by another status check." msgstr "" -msgid "StatusCheck|Invoke an external API as part of the approvals" -msgstr "" - msgid "StatusCheck|Invoke an external API as part of the pipeline process." msgstr "" @@ -32383,9 +32371,6 @@ msgstr "" msgid "The character highlighter helps you keep the subject line to %{titleLength} characters and wrap the body at %{bodyLength} so they are readable in git." msgstr "" -msgid "The coding stage shows the time from the first commit to creating the merge request. The data will automatically be added here once you create your first merge request." -msgstr "" - msgid "The collection of events added to the data gathered for that stage." msgstr "" @@ -32547,9 +32532,6 @@ msgstr "" msgid "The invitation was successfully resent." msgstr "" -msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage." -msgstr "" - msgid "The issue was successfully promoted to an epic. Redirecting to epic..." msgstr "" @@ -32649,9 +32631,6 @@ msgstr "" msgid "The pipelines schedule runs pipelines in the future, repeatedly, for specific branches or tags. Those scheduled pipelines will inherit limited project access based on their associated user." msgstr "" -msgid "The planning stage shows the time from the previous step to pushing your first commit. This time will be added automatically once you push your first commit." -msgstr "" - msgid "The private key to use when a client certificate is provided. This value is encrypted at rest." msgstr "" @@ -32715,9 +32694,6 @@ msgstr "" msgid "The repository must be accessible over %{code_open}http://%{code_close}, %{code_open}https://%{code_close}, %{code_open}ssh://%{code_close} or %{code_open}git://%{code_close}." msgstr "" -msgid "The review stage shows the time from creating the merge request to merging it. The data will automatically be added after you merge your first merge request." -msgstr "" - msgid "The same shared runner executes code from multiple projects, unless you configure autoscaling with %{link} set to 1 (which it is on GitLab.com)." msgstr "" @@ -32742,9 +32718,6 @@ msgstr "" msgid "The specified tab is invalid, please select another" msgstr "" -msgid "The staging stage shows the time between merging the MR and deploying code to the production environment. The data will be automatically added once you deploy to production for the first time." -msgstr "" - msgid "The start date must be ealier than the end date." msgstr "" @@ -32757,9 +32730,6 @@ msgstr "" msgid "The tag name can't be changed for an existing release." msgstr "" -msgid "The testing stage shows the time GitLab CI takes to run every pipeline for the related merge request. The data will automatically be added after your first pipeline finishes running." -msgstr "" - msgid "The time taken by each data entry gathered by that stage." msgstr "" diff --git a/spec/lib/gitlab/ci/templates/templates_spec.rb b/spec/lib/gitlab/ci/templates/templates_spec.rb index dc94b9967ec..2e6df7da232 100644 --- a/spec/lib/gitlab/ci/templates/templates_spec.rb +++ b/spec/lib/gitlab/ci/templates/templates_spec.rb @@ -7,9 +7,10 @@ RSpec.describe 'CI YML Templates' do let(:all_templates) { Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name) } let(:excluded_templates) do - all_templates.select do |name| + excluded = all_templates.select do |name| Gitlab::Template::GitlabCiYmlTemplate.excluded_patterns.any? { |pattern| pattern.match?(name) } end + excluded + ["Terraform.gitlab-ci.yml"] end before do diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 554fd4d4fb0..c4b1e2133ed 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -711,11 +711,7 @@ module Ci end end - context 'when ci_register_job_service_one_by_one is enabled' do - before do - stub_feature_flags(ci_register_job_service_one_by_one: true) - end - + context 'when a long queue is created' do it 'picks builds one-by-one' do expect(Ci::Build).to receive(:find).with(pending_job.id).and_call_original @@ -725,14 +721,6 @@ module Ci include_examples 'handles runner assignment' end - context 'when ci_register_job_service_one_by_one is disabled' do - before do - stub_feature_flags(ci_register_job_service_one_by_one: false) - end - - include_examples 'handles runner assignment' - end - context 'when joining with pending builds table' do before do stub_feature_flags(ci_pending_builds_queue_join: true) diff --git a/spec/services/projects/create_service_spec.rb b/spec/services/projects/create_service_spec.rb index cc2a9e4af9b..ac0b6cc8ef1 100644 --- a/spec/services/projects/create_service_spec.rb +++ b/spec/services/projects/create_service_spec.rb @@ -837,7 +837,7 @@ RSpec.describe Projects::CreateService, '#execute' do expect(AuthorizedProjectUpdate::ProjectCreateWorker).to( receive(:perform_async).and_call_original ) - expect(AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker).to( + expect(AuthorizedProjectUpdate::UserRefreshFromReplicaWorker).to( receive(:bulk_perform_in) .with(1.hour, array_including([user.id], [other_user.id]), diff --git a/spec/services/projects/group_links/create_service_spec.rb b/spec/services/projects/group_links/create_service_spec.rb index 7e617a3568e..9bc780fe177 100644 --- a/spec/services/projects/group_links/create_service_spec.rb +++ b/spec/services/projects/group_links/create_service_spec.rb @@ -54,7 +54,7 @@ RSpec.describe Projects::GroupLinks::CreateService, '#execute' do .with(project.id, group.id, group_access) .and_call_original ) - expect(AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker).to( + expect(AuthorizedProjectUpdate::UserRefreshFromReplicaWorker).to( receive(:bulk_perform_in) .with(1.hour, array_including([user.id], [other_user.id]), diff --git a/spec/services/projects/group_links/destroy_service_spec.rb b/spec/services/projects/group_links/destroy_service_spec.rb index 8436bcb3770..d60e9a01e54 100644 --- a/spec/services/projects/group_links/destroy_service_spec.rb +++ b/spec/services/projects/group_links/destroy_service_spec.rb @@ -31,8 +31,8 @@ RSpec.describe Projects::GroupLinks::DestroyService, '#execute' do subject.execute(group_link) end - it 'calls AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker with a delay to update project authorizations' do - expect(AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker).to( + it 'calls AuthorizedProjectUpdate::UserRefreshFromReplicaWorker with a delay to update project authorizations' do + expect(AuthorizedProjectUpdate::UserRefreshFromReplicaWorker).to( receive(:bulk_perform_in) .with(1.hour, [[user.id]], diff --git a/spec/services/user_project_access_changed_service_spec.rb b/spec/services/user_project_access_changed_service_spec.rb index 28d9fa85c7e..4723619afd2 100644 --- a/spec/services/user_project_access_changed_service_spec.rb +++ b/spec/services/user_project_access_changed_service_spec.rb @@ -19,7 +19,7 @@ RSpec.describe UserProjectAccessChangedService do end it 'permits low-priority operation' do - expect(AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker).to( + expect(AuthorizedProjectUpdate::UserRefreshFromReplicaWorker).to( receive(:bulk_perform_in).with( described_class::DELAY, [[1], [2]], diff --git a/spec/views/projects/services/_form.haml_spec.rb b/spec/views/projects/services/_form.haml_spec.rb index 8dd8bd98388..f063e73dae4 100644 --- a/spec/views/projects/services/_form.haml_spec.rb +++ b/spec/views/projects/services/_form.haml_spec.rb @@ -16,7 +16,7 @@ RSpec.describe 'projects/services/_form' do can?: true, current_application_settings: Gitlab::CurrentSettings.current_application_settings, integration: project.redmine_service, - request: double(referrer: '/services') + request: double(referer: '/services') ) end diff --git a/spec/workers/authorized_project_update/user_refresh_from_replica_worker_spec.rb b/spec/workers/authorized_project_update/user_refresh_from_replica_worker_spec.rb new file mode 100644 index 00000000000..cdf2cb493b0 --- /dev/null +++ b/spec/workers/authorized_project_update/user_refresh_from_replica_worker_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe AuthorizedProjectUpdate::UserRefreshFromReplicaWorker do + it 'is labeled as low urgency' do + expect(described_class.get_urgency).to eq(:low) + end + + it_behaves_like "refreshes user's project authorizations" +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index fccf1256683..34d42addef3 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -130,6 +130,7 @@ RSpec.describe 'Every Sidekiq worker' do 'AuthorizedProjectUpdate::ProjectGroupLinkCreateWorker' => 3, 'AuthorizedProjectUpdate::UserRefreshOverUserRangeWorker' => 3, 'AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker' => 3, + 'AuthorizedProjectUpdate::UserRefreshFromReplicaWorker' => 3, 'AuthorizedProjectsWorker' => 3, 'AutoDevops::DisableWorker' => 3, 'AutoMergeProcessWorker' => 3,