diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 2b5627022fd..855ef1b74ba 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -518670d57d1a6527aaf46b5b9bf5cb00f2e8f11b +f87bc1e983d11788fdbce953dced45ec5554af23 diff --git a/app/assets/javascripts/packages_and_registries/shared/components/registry_list.vue b/app/assets/javascripts/packages_and_registries/shared/components/registry_list.vue new file mode 100644 index 00000000000..58350489435 --- /dev/null +++ b/app/assets/javascripts/packages_and_registries/shared/components/registry_list.vue @@ -0,0 +1,124 @@ + + + diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 66283a03314..277e0ccbf23 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,6 +10,7 @@ module Ci include Presentable include Importable include Ci::HasRef + extend ::Gitlab::Utils::Override BuildArchivedError = Class.new(StandardError) @@ -723,6 +724,14 @@ module Ci self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end + # acts_as_taggable uses this method create/remove tags with contexts + # defined by taggings and to get those contexts it executes a query. + # We don't use any other contexts except `tags`, so we don't need it. + override :custom_contexts + def custom_contexts + [] + end + def tag_list if tags.loaded? tags.map(&:name) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d75f7984e2c..5d3a5ce9b21 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -217,6 +217,10 @@ class CommitStatus < Ci::ApplicationRecord false end + def self.bulk_insert_tags!(statuses, tag_list_by_build) + Gitlab::Ci::Tags::BulkInsert.new(statuses, tag_list_by_build).insert! + end + def locking_enabled? will_save_change_to_status? end diff --git a/app/views/admin/application_settings/_snowplow.html.haml b/app/views/admin/application_settings/_snowplow.html.haml index 756c0e770a6..f7a6a26c645 100644 --- a/app/views/admin/application_settings/_snowplow.html.haml +++ b/app/views/admin/application_settings/_snowplow.html.haml @@ -1,5 +1,5 @@ - expanded = integration_expanded?('snowplow_') -%section.settings.as-snowplow.no-animate#js-snowplow-settings{ class: ('expanded' if expanded) } +%section.settings.as-snowplow.no-animate#js-snowplow-settings{ class: ('expanded' if expanded), data: { qa_selector: 'snowplow_settings_content' } } .settings-header %h4 = _('Snowplow') @@ -15,7 +15,7 @@ %fieldset .form-group .form-check - = f.check_box :snowplow_enabled, class: 'form-check-input' + = f.check_box :snowplow_enabled, class: 'form-check-input', data: { qa_selector: 'snowplow_enabled_checkbox' } = f.label :snowplow_enabled, _('Enable Snowplow tracking'), class: 'form-check-label' .form-group = f.label :snowplow_collector_hostname, _('Collector hostname'), class: 'label-light' @@ -33,4 +33,4 @@ .form-text.text-muted = _('The Snowplow cookie domain.') - = f.submit _('Save changes'), class: 'gl-button btn btn-confirm' + = f.submit _('Save changes'), class: 'gl-button btn btn-confirm', data: { qa_selector: 'save_changes_button' } diff --git a/app/views/profiles/show.html.haml b/app/views/profiles/show.html.haml index b1470520eea..f3993ad8c33 100644 --- a/app/views/profiles/show.html.haml +++ b/app/views/profiles/show.html.haml @@ -116,7 +116,7 @@ %h5= _('Private profile') .checkbox-icon-inline-wrapper - private_profile_label = capture do - = s_("Profiles|Don't display activity-related personal information on your profiles") + = s_("Profiles|Don't display activity-related personal information on your profile") = f.check_box :private_profile, label: private_profile_label, inline: true, wrapper_class: 'mr-0' = link_to sprite_icon('question-o'), help_page_path('user/profile/index.md', anchor: 'make-your-user-profile-page-private') %h5= s_("Profiles|Private contributions") diff --git a/app/views/projects/_new_project_fields.html.haml b/app/views/projects/_new_project_fields.html.haml index c21240b340c..c7648f2e79b 100644 --- a/app/views/projects/_new_project_fields.html.haml +++ b/app/views/projects/_new_project_fields.html.haml @@ -49,7 +49,7 @@ .gl-alert.gl-alert-success.gl-mb-4.gl-display-none.js-user-readme-repo = sprite_icon('check-circle', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') .gl-alert-body - - help_link_start = ''.html_safe % { url: help_page_path('user/profile/index', anchor: 'user-profile-readme') } + - help_link_start = ''.html_safe % { url: help_page_path('user/profile/index', anchor: 'add-details-to-your-profile-with-a-readme') } = html_escape(_('%{project_path} is a project that you can use to add a README to your GitLab profile. Create a public project and initialize the repository with a README to get started. %{help_link_start}Learn more.%{help_link_end}')) % { project_path: "#{current_user.username} / #{current_user.username}".html_safe, help_link_start: help_link_start, help_link_end: ''.html_safe } .form-group diff --git a/config/feature_flags/development/ci_bulk_insert_tags.yml b/config/feature_flags/development/ci_bulk_insert_tags.yml new file mode 100644 index 00000000000..6b8ad4ef39d --- /dev/null +++ b/config/feature_flags/development/ci_bulk_insert_tags.yml @@ -0,0 +1,8 @@ +--- +name: ci_bulk_insert_tags +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73198 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346124 +milestone: '14.6' +type: development +group: group::pipeline execution +default_enabled: false diff --git a/config/metrics/counts_28d/20210216184502_p_ci_templates_implicit_auto_devops_build_monthly.yml b/config/metrics/counts_28d/20210216184502_p_ci_templates_implicit_auto_devops_build_monthly.yml index 6e944dce726..407768232a5 100644 --- a/config/metrics/counts_28d/20210216184502_p_ci_templates_implicit_auto_devops_build_monthly.yml +++ b/config/metrics/counts_28d/20210216184502_p_ci_templates_implicit_auto_devops_build_monthly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184506_p_ci_templates_implicit_auto_devops_deploy_monthly.yml b/config/metrics/counts_28d/20210216184506_p_ci_templates_implicit_auto_devops_deploy_monthly.yml index 940ef5deb65..291ee7eb149 100644 --- a/config/metrics/counts_28d/20210216184506_p_ci_templates_implicit_auto_devops_deploy_monthly.yml +++ b/config/metrics/counts_28d/20210216184506_p_ci_templates_implicit_auto_devops_deploy_monthly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184517_p_ci_templates_5_min_production_app_monthly.yml b/config/metrics/counts_28d/20210216184517_p_ci_templates_5_min_production_app_monthly.yml index 0ae42fcecc0..155994cb6f8 100644 --- a/config/metrics/counts_28d/20210216184517_p_ci_templates_5_min_production_app_monthly.yml +++ b/config/metrics/counts_28d/20210216184517_p_ci_templates_5_min_production_app_monthly.yml @@ -7,7 +7,8 @@ product_stage: deploy product_group: group::5-min-app product_category: five_minute_production_app value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184526_p_ci_templates_aws_cf_deploy_ec2_monthly.yml b/config/metrics/counts_28d/20210216184526_p_ci_templates_aws_cf_deploy_ec2_monthly.yml index a87e1224745..7d3462cb068 100644 --- a/config/metrics/counts_28d/20210216184526_p_ci_templates_aws_cf_deploy_ec2_monthly.yml +++ b/config/metrics/counts_28d/20210216184526_p_ci_templates_aws_cf_deploy_ec2_monthly.yml @@ -8,7 +8,8 @@ product_stage: release product_group: group::release product_category: continuous_delivery value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184534_p_ci_templates_auto_devops_build_monthly.yml b/config/metrics/counts_28d/20210216184534_p_ci_templates_auto_devops_build_monthly.yml index d6333260017..5b95b20f38d 100644 --- a/config/metrics/counts_28d/20210216184534_p_ci_templates_auto_devops_build_monthly.yml +++ b/config/metrics/counts_28d/20210216184534_p_ci_templates_auto_devops_build_monthly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184538_p_ci_templates_auto_devops_deploy_monthly.yml b/config/metrics/counts_28d/20210216184538_p_ci_templates_auto_devops_deploy_monthly.yml index 792d41e3bbd..6747c81367c 100644 --- a/config/metrics/counts_28d/20210216184538_p_ci_templates_auto_devops_deploy_monthly.yml +++ b/config/metrics/counts_28d/20210216184538_p_ci_templates_auto_devops_deploy_monthly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210216184542_p_ci_templates_auto_devops_deploy_latest_monthly.yml b/config/metrics/counts_28d/20210216184542_p_ci_templates_auto_devops_deploy_latest_monthly.yml index deafb216e99..9efd5df6104 100644 --- a/config/metrics/counts_28d/20210216184542_p_ci_templates_auto_devops_deploy_latest_monthly.yml +++ b/config/metrics/counts_28d/20210216184542_p_ci_templates_auto_devops_deploy_latest_monthly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 28d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_28d/20210902000813_p_ci_templates_implicit_auto_devops_deploy_latest_monthly.yml b/config/metrics/counts_28d/20210902000813_p_ci_templates_implicit_auto_devops_deploy_latest_monthly.yml index 3b667e49f0d..3450683aaf3 100644 --- a/config/metrics/counts_28d/20210902000813_p_ci_templates_implicit_auto_devops_deploy_latest_monthly.yml +++ b/config/metrics/counts_28d/20210902000813_p_ci_templates_implicit_auto_devops_deploy_latest_monthly.yml @@ -6,8 +6,9 @@ product_stage: '' product_group: '' product_category: '' value_type: number -status: active +status: removed milestone: '14.3' +milestone_removed: '14.6' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: 28d data_source: redis_hll diff --git a/config/metrics/counts_7d/20210216184500_p_ci_templates_implicit_auto_devops_build_weekly.yml b/config/metrics/counts_7d/20210216184500_p_ci_templates_implicit_auto_devops_build_weekly.yml index fbfb93ded5a..9bec574722c 100644 --- a/config/metrics/counts_7d/20210216184500_p_ci_templates_implicit_auto_devops_build_weekly.yml +++ b/config/metrics/counts_7d/20210216184500_p_ci_templates_implicit_auto_devops_build_weekly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 7d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_7d/20210216184504_p_ci_templates_implicit_auto_devops_deploy_weekly.yml b/config/metrics/counts_7d/20210216184504_p_ci_templates_implicit_auto_devops_deploy_weekly.yml index 0b05e776c70..ec176ce689c 100644 --- a/config/metrics/counts_7d/20210216184504_p_ci_templates_implicit_auto_devops_deploy_weekly.yml +++ b/config/metrics/counts_7d/20210216184504_p_ci_templates_implicit_auto_devops_deploy_weekly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 7d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_7d/20210216184515_p_ci_templates_5_min_production_app_weekly.yml b/config/metrics/counts_7d/20210216184515_p_ci_templates_5_min_production_app_weekly.yml index 45fe5e380f5..050ad56eb91 100644 --- a/config/metrics/counts_7d/20210216184515_p_ci_templates_5_min_production_app_weekly.yml +++ b/config/metrics/counts_7d/20210216184515_p_ci_templates_5_min_production_app_weekly.yml @@ -7,7 +7,8 @@ product_stage: deploy product_group: group::5-min-app product_category: five_minute_production_app value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 7d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_7d/20210216184524_p_ci_templates_aws_cf_deploy_ec2_weekly.yml b/config/metrics/counts_7d/20210216184524_p_ci_templates_aws_cf_deploy_ec2_weekly.yml index adc12342146..0477d58a5d8 100644 --- a/config/metrics/counts_7d/20210216184524_p_ci_templates_aws_cf_deploy_ec2_weekly.yml +++ b/config/metrics/counts_7d/20210216184524_p_ci_templates_aws_cf_deploy_ec2_weekly.yml @@ -8,7 +8,8 @@ product_stage: release product_group: group::release product_category: continuous_delivery value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 7d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_7d/20210216184536_p_ci_templates_auto_devops_deploy_weekly.yml b/config/metrics/counts_7d/20210216184536_p_ci_templates_auto_devops_deploy_weekly.yml index b01a0288228..b06a4fa5577 100644 --- a/config/metrics/counts_7d/20210216184536_p_ci_templates_auto_devops_deploy_weekly.yml +++ b/config/metrics/counts_7d/20210216184536_p_ci_templates_auto_devops_deploy_weekly.yml @@ -7,8 +7,9 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed milestone: '14.3' +milestone_removed: '14.6' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: 7d data_source: redis_hll diff --git a/config/metrics/counts_7d/20210216184540_p_ci_templates_auto_devops_deploy_latest_weekly.yml b/config/metrics/counts_7d/20210216184540_p_ci_templates_auto_devops_deploy_latest_weekly.yml index 59f9f25aa06..d7278e2dd34 100644 --- a/config/metrics/counts_7d/20210216184540_p_ci_templates_auto_devops_deploy_latest_weekly.yml +++ b/config/metrics/counts_7d/20210216184540_p_ci_templates_auto_devops_deploy_latest_weekly.yml @@ -7,7 +7,8 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed +milestone_removed: '14.6' time_frame: 7d data_source: redis_hll instrumentation_class: RedisHLLMetric diff --git a/config/metrics/counts_7d/20210902000809_p_ci_templates_implicit_auto_devops_deploy_latest_weekly.yml b/config/metrics/counts_7d/20210902000809_p_ci_templates_implicit_auto_devops_deploy_latest_weekly.yml index 7f674324a31..beb82977e3f 100644 --- a/config/metrics/counts_7d/20210902000809_p_ci_templates_implicit_auto_devops_deploy_latest_weekly.yml +++ b/config/metrics/counts_7d/20210902000809_p_ci_templates_implicit_auto_devops_deploy_latest_weekly.yml @@ -6,8 +6,9 @@ product_stage: configure product_group: group::configure product_category: infrastructure_as_code value_type: number -status: active +status: removed milestone: '14.3' +milestone_removed: '14.6' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69204 time_frame: 7d data_source: redis_hll diff --git a/db/migrate/20211123182614_make_iteration_cadences_start_date_nullable.rb b/db/migrate/20211123182614_make_iteration_cadences_start_date_nullable.rb new file mode 100644 index 00000000000..10a0c6ca402 --- /dev/null +++ b/db/migrate/20211123182614_make_iteration_cadences_start_date_nullable.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class MakeIterationCadencesStartDateNullable < Gitlab::Database::Migration[1.0] + def change + change_column_null :iterations_cadences, :start_date, true + end +end diff --git a/db/schema_migrations/20211123182614 b/db/schema_migrations/20211123182614 new file mode 100644 index 00000000000..8b67ec7cd26 --- /dev/null +++ b/db/schema_migrations/20211123182614 @@ -0,0 +1 @@ +9a3ba69a1df02059b240393cc381c4a5ba9db0f116818aa9f3d4f1009f055b09 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 1086961ab41..32baa151239 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -15439,7 +15439,7 @@ CREATE TABLE iterations_cadences ( group_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, - start_date date NOT NULL, + start_date date, last_run_date date, duration_in_weeks integer, iterations_in_advance integer, diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index e7504963526..8e45da56fe8 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -873,7 +873,7 @@ project.container_repositories.find_each do |repo| puts repo.attributes # Start the tag cleanup - puts Projects::ContainerRepository::CleanupTagsService.new(project, user, policy.attributes.except("created_at", "updated_at")).execute(repo) + puts Projects::ContainerRepository::CleanupTagsService.new(repo, user, policy.attributes.except("created_at", "updated_at")).execute() end ``` diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md index 65a8b4c1cad..a501471c816 100644 --- a/doc/development/service_ping/implement.md +++ b/doc/development/service_ping/implement.md @@ -26,6 +26,10 @@ To implement a new metric in Service Ping, follow these steps: 1. [Verify your metric](#verify-your-metric) 1. [Set up and test Service Ping locally](#set-up-and-test-service-ping-locally) +NOTE: +When you add or change a Service Metric, you must migrate metrics to [instrumentation classes](metrics_instrumentation.md). +For information about the progress on migrating Service ping metrics, see this [epic](https://gitlab.com/groups/gitlab-org/-/epics/5547). + ## Instrumentation classes We recommend you use [instrumentation classes](metrics_instrumentation.md) in `usage_data.rb` where possible. diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index e3ccbf98d30..6274f91e818 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -100,17 +100,17 @@ When visiting the public page of a user, you can only see the projects which you If the [public level is restricted](../admin_area/settings/visibility_and_access_controls.md#restrict-visibility-levels), user profiles are only visible to signed-in users. -## User profile README +## Add details to your profile with a README > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232157) in GitLab 14.5. -You can add a README section to your profile that can include more information and [formatting](../markdown.md) than -your profile's bio. +If you want to add more information to your profile page, you can create a README file. When you populate the README file with information, it's included on your profile page. To add a README to your profile: 1. Create a new public project with the same project path as your GitLab username. 1. Create a README file inside this project. The file can be any valid [README or index file](../project/repository/index.md#readme-and-index-files). +1. Populate the README file with [Markdown](../markdown.md). To use an existing project, [update the path](../project/settings/index.md#renaming-a-repository) of the project to match your username. diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 9dba557eef6..15b0ff3c04d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,11 +6,17 @@ module Gitlab module Chain class Create < Chain::Base include Chain::Helpers + include Gitlab::Utils::StrongMemoize def perform! logger.instrument(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - pipeline.save! + tags = extract_tag_list_by_status + + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + end end end rescue ActiveRecord::RecordInvalid => e @@ -20,6 +26,37 @@ module Gitlab def break? !pipeline.persisted? end + + private + + def statuses + strong_memoize(:statuses) do + pipeline.stages.flat_map(&:statuses) + end + end + + # We call `job.tag_list=` to assign tags to the jobs from the + # Chain::Seed step which uses the `@tag_list` instance variable to + # store them on the record. We remove them here because we want to + # bulk insert them, otherwise they would be inserted and assigned one + # by one with callbacks. We must use `remove_instance_variable` + # because having the instance variable defined would still run the callbacks + def extract_tag_list_by_status + return {} unless bulk_insert_tags? + + statuses.each.with_object({}) do |job, acc| + tag_list = job.clear_memoization(:tag_list) + next unless tag_list + + acc[job.name] = tag_list + end + end + + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index c1e3a4ec3f7..bb8831095e4 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -200,11 +200,13 @@ module Gitlab end def runner_tags - { tag_list: evaluate_runner_tags }.compact + strong_memoize(:runner_tags) do + { tag_list: evaluate_runner_tags }.compact + end end def evaluate_runner_tags - @seed_attributes[:tag_list]&.map do |tag| + @seed_attributes.delete(:tag_list)&.map do |tag| ExpandVariables.expand_existing(tag, -> { evaluate_context.variables_hash }) end end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb new file mode 100644 index 00000000000..a299df7e2d9 --- /dev/null +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Tags + class BulkInsert + TAGGINGS_BATCH_SIZE = 1000 + TAGS_BATCH_SIZE = 500 + + def initialize(statuses, tag_list_by_status) + @statuses = statuses + @tag_list_by_status = tag_list_by_status + end + + def insert! + return false if tag_list_by_status.empty? + + persist_build_tags! + end + + private + + attr_reader :statuses, :tag_list_by_status + + def persist_build_tags! + all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + tag_records_by_name = create_tags(all_tags).index_by(&:name) + taggings = build_taggings_attributes(tag_records_by_name) + + return false if taggings.empty? + + taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| + ActsAsTaggableOn::Tagging.insert_all!(taggings) + end + + true + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_tags(tags) + existing_tag_records = ActsAsTaggableOn::Tag.where(name: tags).to_a + missing_tags = detect_missing_tags(tags, existing_tag_records) + return existing_tag_records if missing_tags.empty? + + missing_tags + .map { |tag| { name: tag } } + .each_slice(TAGS_BATCH_SIZE) do |tags_attributes| + ActsAsTaggableOn::Tag.insert_all!(tags_attributes) + end + + ActsAsTaggableOn::Tag.where(name: tags).to_a + end + # rubocop: enable CodeReuse/ActiveRecord + + def build_taggings_attributes(tag_records_by_name) + taggings = statuses.flat_map do |status| + tag_list = tag_list_by_status[status.name] + next unless tag_list + + tags = tag_records_by_name.values_at(*tag_list) + taggings_for(tags, status) + end + + taggings.compact! + taggings + end + + def taggings_for(tags, status) + tags.map do |tag| + { + tag_id: tag.id, + taggable_type: CommitStatus.name, + taggable_id: status.id, + created_at: Time.current, + context: 'tags' + } + end + end + + def detect_missing_tags(tags, tag_records) + if tags.size != tag_records.size + tags - tag_records.map(&:name) + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 17ed1d2e87f..d32444833fb 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -9,6 +9,7 @@ pages: script: - mkdir .public - cp -r * .public + - rm -rf public - mv .public public artifacts: paths: diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 57bedcaf551..5cd517faa08 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -11174,6 +11174,9 @@ msgstr "" msgid "Delete Key" msgstr "" +msgid "Delete Selected" +msgstr "" + msgid "Delete Value Stream" msgstr "" @@ -26597,7 +26600,7 @@ msgstr "" msgid "Profiles|Do not show on profile" msgstr "" -msgid "Profiles|Don't display activity-related personal information on your profiles" +msgid "Profiles|Don't display activity-related personal information on your profile" msgstr "" msgid "Profiles|Edit Profile" diff --git a/qa/qa/flow/settings.rb b/qa/qa/flow/settings.rb new file mode 100644 index 00000000000..775b7686c10 --- /dev/null +++ b/qa/qa/flow/settings.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module QA + module Flow + module Settings + module_function + + def disable_snowplow + Flow::Login.while_signed_in_as_admin do + QA::Page::Main::Menu.perform(&:go_to_admin_area) + QA::Page::Admin::Menu.perform(&:go_to_general_settings) + QA::Page::Admin::Settings::Component::Snowplow.perform(&:disable_snowplow_tracking) + end + end + + def enable_snowplow + Flow::Login.while_signed_in_as_admin do + QA::Page::Main::Menu.perform(&:go_to_admin_area) + QA::Page::Admin::Menu.perform(&:go_to_general_settings) + QA::Page::Admin::Settings::Component::Snowplow.perform(&:enable_snowplow_tracking) + end + end + end + end +end diff --git a/qa/qa/page/admin/settings/component/snowplow.rb b/qa/qa/page/admin/settings/component/snowplow.rb new file mode 100644 index 00000000000..e05679feac3 --- /dev/null +++ b/qa/qa/page/admin/settings/component/snowplow.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module QA + module Page + module Admin + module Settings + module Component + class Snowplow < Page::Base + include QA::Page::Settings::Common + + view 'app/views/admin/application_settings/_snowplow.html.haml' do + element :snowplow_settings_content + element :snowplow_enabled_checkbox + element :save_changes_button + end + + def enable_snowplow_tracking + expand_content(:snowplow_settings_content) do + check_snowplow_enabled_checkbox + click_save_changes_button + end + end + + def disable_snowplow_tracking + expand_content(:snowplow_settings_content) do + uncheck_snowplow_enabled_checkbox + click_save_changes_button + end + end + + private + + def check_snowplow_enabled_checkbox + check_element(:snowplow_enabled_checkbox) + end + + def uncheck_snowplow_enabled_checkbox + uncheck_element(:snowplow_enabled_checkbox) + end + + def click_save_changes_button + click_element :save_changes_button + end + end + end + end + end + end +end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index fb86f4672bc..981f10e8260 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -190,7 +190,7 @@ FactoryBot.define do end after :create do |project, evaluator| - raise "Failed to create repository!" unless project.create_repository + raise "Failed to create repository!" unless project.repository.exists? || project.create_repository evaluator.files.each do |filename, content| project.repository.create_file( diff --git a/spec/frontend/packages_and_registries/shared/components/registry_list_spec.js b/spec/frontend/packages_and_registries/shared/components/registry_list_spec.js new file mode 100644 index 00000000000..aaca58d21bb --- /dev/null +++ b/spec/frontend/packages_and_registries/shared/components/registry_list_spec.js @@ -0,0 +1,199 @@ +import { GlButton, GlFormCheckbox, GlKeysetPagination } from '@gitlab/ui'; +import { nextTick } from 'vue'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import component from '~/packages_and_registries/shared/components/registry_list.vue'; + +describe('Registry List', () => { + let wrapper; + + const items = [{ id: 'a' }, { id: 'b' }]; + const defaultPropsData = { + title: 'test_title', + items, + }; + + const rowScopedSlot = ` +
+ + {{props.first}} +

{{props.isSelected(props.item)}}

+
`; + + const mountComponent = ({ propsData = defaultPropsData } = {}) => { + wrapper = shallowMountExtended(component, { + propsData, + scopedSlots: { + default: rowScopedSlot, + }, + }); + }; + + const findSelectAll = () => wrapper.findComponent(GlFormCheckbox); + const findDeleteSelected = () => wrapper.findComponent(GlButton); + const findPagination = () => wrapper.findComponent(GlKeysetPagination); + const findScopedSlots = () => wrapper.findAllByTestId('scoped-slot'); + const findScopedSlotSelectButton = (index) => findScopedSlots().at(index).find('button'); + const findScopedSlotFirstValue = (index) => findScopedSlots().at(index).find('span'); + const findScopedSlotIsSelectedValue = (index) => findScopedSlots().at(index).find('p'); + + afterEach(() => { + wrapper.destroy(); + }); + + describe('header', () => { + it('renders the title passed in the prop', () => { + mountComponent(); + + expect(wrapper.text()).toContain(defaultPropsData.title); + }); + + describe('select all checkbox', () => { + beforeEach(() => { + mountComponent(); + }); + + it('exists', () => { + expect(findSelectAll().exists()).toBe(true); + }); + + it('select and unselect all', async () => { + // no row is not selected + items.forEach((item, index) => { + expect(findScopedSlotIsSelectedValue(index).text()).toBe(''); + }); + + // simulate selection + findSelectAll().vm.$emit('input', true); + await nextTick(); + + // all rows selected + items.forEach((item, index) => { + expect(findScopedSlotIsSelectedValue(index).text()).toBe('true'); + }); + + // simulate de-selection + findSelectAll().vm.$emit('input', ''); + await nextTick(); + + // no row is not selected + items.forEach((item, index) => { + expect(findScopedSlotIsSelectedValue(index).text()).toBe(''); + }); + }); + }); + + describe('delete button', () => { + it('has the correct text', () => { + mountComponent(); + + expect(findDeleteSelected().text()).toBe(component.i18n.deleteSelected); + }); + + it('is hidden when hiddenDelete is true', () => { + mountComponent({ propsData: { ...defaultPropsData, hiddenDelete: true } }); + + expect(findDeleteSelected().exists()).toBe(false); + }); + + it('is disabled when isLoading is true', () => { + mountComponent({ propsData: { ...defaultPropsData, isLoading: true } }); + + expect(findDeleteSelected().props('disabled')).toBe(true); + }); + + it('is disabled when no row is selected', async () => { + mountComponent(); + + expect(findDeleteSelected().props('disabled')).toBe(true); + + await findScopedSlotSelectButton(0).trigger('click'); + + expect(findDeleteSelected().props('disabled')).toBe(false); + }); + + it('on click emits the delete event with the selected rows', async () => { + mountComponent(); + + await findScopedSlotSelectButton(0).trigger('click'); + + findDeleteSelected().vm.$emit('click'); + + expect(wrapper.emitted('delete')).toEqual([[[items[0]]]]); + }); + }); + }); + + describe('main area', () => { + beforeEach(() => { + mountComponent(); + }); + + it('renders scopedSlots based on the items props', () => { + expect(findScopedSlots()).toHaveLength(items.length); + }); + + it('populates the scope of the slot correctly', async () => { + expect(findScopedSlots().at(0).exists()).toBe(true); + + // it's the first slot + expect(findScopedSlotFirstValue(0).text()).toBe('true'); + + // item is not selected, falsy is translated to empty string + expect(findScopedSlotIsSelectedValue(0).text()).toBe(''); + + // find the button with the bound function + await findScopedSlotSelectButton(0).trigger('click'); + + // the item is selected + expect(findScopedSlotIsSelectedValue(0).text()).toBe('true'); + }); + }); + + describe('footer', () => { + let pagination; + + beforeEach(() => { + pagination = { hasPreviousPage: false, hasNextPage: true }; + }); + + it('has a pagination', () => { + mountComponent({ + propsData: { ...defaultPropsData, pagination }, + }); + + expect(findPagination().props()).toMatchObject(pagination); + }); + + it.each` + hasPreviousPage | hasNextPage | visible + ${true} | ${true} | ${true} + ${true} | ${false} | ${true} + ${false} | ${true} | ${true} + ${false} | ${false} | ${false} + `( + 'when hasPreviousPage is $hasPreviousPage and hasNextPage is $hasNextPage is $visible that the pagination is shown', + ({ hasPreviousPage, hasNextPage, visible }) => { + pagination = { hasPreviousPage, hasNextPage }; + mountComponent({ + propsData: { ...defaultPropsData, pagination }, + }); + + expect(findPagination().exists()).toBe(visible); + }, + ); + + it('pagination emits the correct events', () => { + mountComponent({ + propsData: { ...defaultPropsData, pagination }, + }); + + findPagination().vm.$emit('prev'); + + expect(wrapper.emitted('prev-page')).toEqual([[]]); + + findPagination().vm.$emit('next'); + + expect(wrapper.emitted('next-page')).toEqual([[]]); + }); + }); +}); diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index d60ecc80a6e..4206483b228 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -56,4 +56,74 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do .to include /Failed to persist the pipeline/ end end + + context 'tags persistence' do + let(:stage) do + build(:ci_stage_entity, pipeline: pipeline) + end + + let(:job) do + build(:ci_build, stage: stage, pipeline: pipeline, project: project) + end + + let(:bridge) do + build(:ci_bridge, stage: stage, pipeline: pipeline, project: project) + end + + before do + pipeline.stages = [stage] + stage.statuses = [job, bridge] + end + + context 'without tags' do + it 'extracts an empty tag list' do + expect(CommitStatus) + .to receive(:bulk_insert_tags!) + .with(stage.statuses, {}) + .and_call_original + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_falsey + expect(job).to be_persisted + expect(job.tag_list).to eq([]) + end + end + + context 'with tags' do + before do + job.tag_list = %w[tag1 tag2] + end + + it 'bulk inserts tags' do + expect(CommitStatus) + .to receive(:bulk_insert_tags!) + .with(stage.statuses, { job.name => %w[tag1 tag2] }) + .and_call_original + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_falsey + expect(job).to be_persisted + expect(job.tag_list).to match_array(%w[tag1 tag2]) + end + end + + context 'when the feature flag is disabled' do + before do + job.tag_list = %w[tag1 tag2] + stub_feature_flags(ci_bulk_insert_tags: false) + end + + it 'follows the old code path' do + expect(CommitStatus).not_to receive(:bulk_insert_tags!) + + step.perform! + + expect(job.instance_variable_defined?(:@tag_list)).to be_truthy + expect(job).to be_persisted + expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) + end + end + end end diff --git a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb new file mode 100644 index 00000000000..6c1f56de840 --- /dev/null +++ b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Tags::BulkInsert do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:pipeline) { create(:ci_pipeline, project: project) } + let_it_be_with_refind(:job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) } + let_it_be_with_refind(:other_job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) } + let_it_be_with_refind(:bridge) { create(:ci_bridge, pipeline: pipeline, project: project) } + + let(:statuses) { [job, bridge, other_job] } + + subject(:service) { described_class.new(statuses, tags_list) } + + describe '#insert!' do + context 'without tags' do + let(:tags_list) { {} } + + it { expect(service.insert!).to be_falsey } + end + + context 'with tags' do + let(:tags_list) do + { + job.name => %w[tag1 tag2], + other_job.name => %w[tag2 tag3 tag4] + } + end + + it 'persists tags' do + expect(service.insert!).to be_truthy + + expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) + expect(other_job.reload.tag_list).to match_array(%w[tag2 tag3 tag4]) + end + end + end +end diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb index 59d14574c02..31421dcf40f 100644 --- a/spec/models/commit_status_spec.rb +++ b/spec/models/commit_status_spec.rb @@ -958,4 +958,21 @@ RSpec.describe CommitStatus do expect(build_from_other_pipeline.reload).to have_attributes(retried: false, processed: false) end end + + describe '.bulk_insert_tags!' do + let(:statuses) { double('statuses') } + let(:tag_list_by_build) { double('tag list') } + let(:inserter) { double('inserter') } + + it 'delegates to bulk insert class' do + expect(Gitlab::Ci::Tags::BulkInsert) + .to receive(:new) + .with(statuses, tag_list_by_build) + .and_return(inserter) + + expect(inserter).to receive(:insert!) + + described_class.bulk_insert_tags!(statuses, tag_list_by_build) + end + end end diff --git a/spec/services/ci/create_pipeline_service/tags_spec.rb b/spec/services/ci/create_pipeline_service/tags_spec.rb index 335d35010c8..cbbeb870c5f 100644 --- a/spec/services/ci/create_pipeline_service/tags_spec.rb +++ b/spec/services/ci/create_pipeline_service/tags_spec.rb @@ -7,16 +7,15 @@ RSpec.describe Ci::CreatePipelineService do let_it_be(:user) { project.owner } let(:ref) { 'refs/heads/master' } - let(:source) { :push } let(:service) { described_class.new(project, user, { ref: ref }) } - let(:pipeline) { service.execute(source).payload } + let(:pipeline) { create_pipeline } before do - stub_ci_pipeline_yaml_file(config) + stub_yaml_config(config) end context 'with valid config' do - let(:config) { YAML.dump({ test: { script: 'ls', tags: %w[tag1 tag2] } }) } + let(:config) { { test: { script: 'ls', tags: %w[tag1 tag2] } } } it 'creates a pipeline', :aggregate_failures do expect(pipeline).to be_created_successfully @@ -25,8 +24,8 @@ RSpec.describe Ci::CreatePipelineService do end context 'with too many tags' do - let(:tags) { Array.new(50) {|i| "tag-#{i}" } } - let(:config) { YAML.dump({ test: { script: 'ls', tags: tags } }) } + let(:tags) { build_tag_list(label: 'custom', size: 50) } + let(:config) { { test: { script: 'ls', tags: tags } } } it 'creates a pipeline without builds', :aggregate_failures do expect(pipeline).not_to be_created_successfully @@ -34,5 +33,167 @@ RSpec.describe Ci::CreatePipelineService do expect(pipeline.yaml_errors).to eq("jobs:test:tags config must be less than the limit of #{Gitlab::Ci::Config::Entry::Tags::TAGS_LIMIT} tags") end end + + context 'tags persistence' do + let(:config) do + { + build: { + script: 'ls', + stage: 'build', + tags: build_tag_list(label: 'build') + }, + test: { + script: 'ls', + stage: 'test', + tags: build_tag_list(label: 'test') + } + } + end + + let(:config_without_tags) do + config.transform_values { |job| job.except(:tags) } + end + + context 'with multiple tags' do + context 'when the tags do not exist' do + it 'does not execute N+1 queries' do + stub_yaml_config(config_without_tags) + + # warm up the cached objects so we get a more accurate count + create_pipeline + + control = ActiveRecord::QueryRecorder.new(skip_cached: false) do + create_pipeline + end + + stub_yaml_config(config) + + # 2 select tags.* + # 1 insert tags + # 1 insert taggings + tags_queries_size = 4 + + expect { pipeline } + .not_to exceed_all_query_limit(control) + .with_threshold(tags_queries_size) + + expect(pipeline).to be_created_successfully + end + end + + context 'when the feature flag is disabled' do + before do + stub_feature_flags(ci_bulk_insert_tags: false) + end + + it 'executes N+1s queries' do + stub_yaml_config(config_without_tags) + + # warm up the cached objects so we get a more accurate count + create_pipeline + + control = ActiveRecord::QueryRecorder.new(skip_cached: false) do + create_pipeline + end + + stub_yaml_config(config) + + expect { pipeline } + .to exceed_all_query_limit(control) + .with_threshold(4) + + expect(pipeline).to be_created_successfully + end + end + + context 'when tags are already persisted' do + it 'does not execute N+1 queries' do + # warm up the cached objects so we get a more accurate count + # and insert the tags + create_pipeline + + control = ActiveRecord::QueryRecorder.new(skip_cached: false) do + create_pipeline + end + + # 1 select tags.* + # 1 insert taggings + tags_queries_size = 2 + + expect { pipeline } + .not_to exceed_all_query_limit(control) + .with_threshold(tags_queries_size) + + expect(pipeline).to be_created_successfully + end + end + end + + context 'with bridge jobs' do + let(:config) do + { + test_1: { + script: 'ls', + stage: 'test', + tags: build_tag_list(label: 'test_1') + }, + test_2: { + script: 'ls', + stage: 'test', + tags: build_tag_list(label: '$CI_JOB_NAME') + }, + test_3: { + script: 'ls', + stage: 'test', + tags: build_tag_list(label: 'test_1') + build_tag_list(label: 'test_2') + }, + test_4: { + script: 'ls', + stage: 'test' + }, + deploy: { + stage: 'deploy', + trigger: 'my/project' + } + } + end + + it do + expect(pipeline).to be_created_successfully + expect(pipeline.bridges.size).to eq(1) + expect(pipeline.builds.size).to eq(4) + + expect(tags_for('test_1')) + .to have_attributes(count: 5) + .and all(match(/test_1-tag-\d+/)) + + expect(tags_for('test_2')) + .to have_attributes(count: 5) + .and all(match(/test_2-tag-\d+/)) + + expect(tags_for('test_3')) + .to have_attributes(count: 10) + .and all(match(/test_[1,2]-tag-\d+/)) + + expect(tags_for('test_4')).to be_empty + end + end + end + end + + def tags_for(build_name) + pipeline.builds.find_by_name(build_name).tag_list + end + + def stub_yaml_config(config) + stub_ci_pipeline_yaml_file(YAML.dump(config)) + end + + def create_pipeline + service.execute(:push).payload + end + + def build_tag_list(label:, size: 5) + Array.new(size) { |index| "#{label}-tag-#{index}" } end end diff --git a/spec/support/database/cross-database-modification-allowlist.yml b/spec/support/database/cross-database-modification-allowlist.yml index badaead950f..1a81ff79973 100644 --- a/spec/support/database/cross-database-modification-allowlist.yml +++ b/spec/support/database/cross-database-modification-allowlist.yml @@ -30,6 +30,7 @@ - "./spec/lib/gitlab/ci/pipeline/chain/create_spec.rb" - "./spec/lib/gitlab/ci/pipeline/chain/seed_block_spec.rb" - "./spec/lib/gitlab/ci/pipeline/seed/build_spec.rb" +- "./spec/lib/gitlab/ci/tags/bulk_insert_spec.rb" - "./spec/lib/gitlab/ci/templates/5_minute_production_app_ci_yaml_spec.rb" - "./spec/lib/gitlab/ci/templates/AWS/deploy_ecs_gitlab_ci_yaml_spec.rb" - "./spec/lib/gitlab/ci/templates/Jobs/deploy_gitlab_ci_yaml_spec.rb"