From 5a9763751b60521b606f2856dfdc1312c8ce416d Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 5 Apr 2022 03:08:30 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../components/invite_modal_base.vue | 72 ++++--- app/models/project.rb | 2 +- .../database/consistency_check_service.rb | 109 ++++++++++ app/workers/all_queues.yml | 18 ++ ...espace_mirrors_consistency_check_worker.rb | 28 +++ ...roject_mirrors_consistency_check_worker.rb | 28 +++ ...ci_namespace_mirrors_consistency_check.yml | 8 + .../ci_project_mirrors_consistency_check.yml | 8 + config/initializers/1_settings.rb | 6 + ..._templates_total_unique_counts_monthly.yml | 1 + ...i_templates_database_liquibase_monthly.yml | 25 +++ ...i_templates_total_unique_counts_weekly.yml | 1 + ...ci_templates_database_liquibase_weekly.yml | 25 +++ ...28_schedule_merge_topics_with_same_name.rb | 23 +++ db/schema_migrations/20220223124428 | 1 + doc/api/project_level_variables.md | 2 +- .../merge_topics_with_same_name.rb | 76 +++++++ .../Database/liquibase.gitlab-ci.yml | 149 ++++++++++++++ lib/gitlab/database/consistency_checker.rb | 122 +++++++++++ .../known_events/ci_templates.yml | 4 + .../formatters/test_stats_formatter_spec.rb | 1 + spec/fixtures/avatars/avatar1.png | Bin 0 -> 1461 bytes spec/fixtures/avatars/avatar2.png | Bin 0 -> 1665 bytes spec/fixtures/avatars/avatar3.png | Bin 0 -> 1767 bytes spec/fixtures/avatars/avatar4.png | Bin 0 -> 1624 bytes spec/fixtures/avatars/avatar5.png | Bin 0 -> 1700 bytes .../components/invite_groups_modal_spec.js | 13 +- .../components/invite_members_modal_spec.js | 25 +-- .../components/invite_modal_base_spec.js | 24 ++- .../application_settings_helper_spec.rb | 15 +- .../merge_topics_with_same_name_spec.rb | 135 +++++++++++++ .../database/consistency_checker_spec.rb | 189 ++++++++++++++++++ ...hedule_merge_topics_with_same_name_spec.rb | 36 ++++ spec/models/project_spec.rb | 4 + .../consistency_check_service_spec.rb | 154 ++++++++++++++ .../_repository_storage.html.haml_spec.rb | 40 ++-- ...e_mirrors_consistency_check_worker_spec.rb | 67 +++++++ ...t_mirrors_consistency_check_worker_spec.rb | 67 +++++++ tooling/danger/product_intelligence.rb | 4 +- 39 files changed, 1391 insertions(+), 91 deletions(-) create mode 100644 app/services/database/consistency_check_service.rb create mode 100644 app/workers/database/ci_namespace_mirrors_consistency_check_worker.rb create mode 100644 app/workers/database/ci_project_mirrors_consistency_check_worker.rb create mode 100644 config/feature_flags/development/ci_namespace_mirrors_consistency_check.yml create mode 100644 config/feature_flags/development/ci_project_mirrors_consistency_check.yml create mode 100644 config/metrics/counts_28d/20220307150912_p_ci_templates_database_liquibase_monthly.yml create mode 100644 config/metrics/counts_7d/20220307150854_p_ci_templates_database_liquibase_weekly.yml create mode 100644 db/post_migrate/20220223124428_schedule_merge_topics_with_same_name.rb create mode 100644 db/schema_migrations/20220223124428 create mode 100644 lib/gitlab/background_migration/merge_topics_with_same_name.rb create mode 100644 lib/gitlab/ci/templates/Database/liquibase.gitlab-ci.yml create mode 100644 lib/gitlab/database/consistency_checker.rb create mode 100644 spec/fixtures/avatars/avatar1.png create mode 100644 spec/fixtures/avatars/avatar2.png create mode 100644 spec/fixtures/avatars/avatar3.png create mode 100644 spec/fixtures/avatars/avatar4.png create mode 100644 spec/fixtures/avatars/avatar5.png create mode 100644 spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb create mode 100644 spec/lib/gitlab/database/consistency_checker_spec.rb create mode 100644 spec/migrations/20220223124428_schedule_merge_topics_with_same_name_spec.rb create mode 100644 spec/services/database/consistency_check_service_spec.rb create mode 100644 spec/workers/database/ci_namespace_mirrors_consistency_check_worker_spec.rb create mode 100644 spec/workers/database/ci_project_mirrors_consistency_check_worker_spec.rb diff --git a/app/assets/javascripts/invite_members/components/invite_modal_base.vue b/app/assets/javascripts/invite_members/components/invite_modal_base.vue index 43cd889c9b3..d9297614a7e 100644 --- a/app/assets/javascripts/invite_members/components/invite_modal_base.vue +++ b/app/assets/javascripts/invite_members/components/invite_modal_base.vue @@ -7,7 +7,6 @@ import { GlDatepicker, GlLink, GlSprintf, - GlButton, GlFormInput, } from '@gitlab/ui'; import { sprintf } from '~/locale'; @@ -41,7 +40,6 @@ export default { GlDropdown, GlDropdownItem, GlSprintf, - GlButton, GlFormInput, ContentTransition, }, @@ -104,6 +102,11 @@ export default { required: false, default: INVITE_BUTTON_TEXT, }, + cancelButtonText: { + type: String, + required: false, + default: CANCEL_BUTTON_TEXT, + }, currentSlot: { type: String, required: false, @@ -114,6 +117,11 @@ export default { required: false, default: () => [], }, + preventCancelDefault: { + type: Boolean, + required: false, + default: false, + }, }, data() { // Be sure to check out reset! @@ -141,6 +149,22 @@ export default { contentSlots() { return [...DEFAULT_SLOTS, ...(this.extraSlots || [])]; }, + actionPrimary() { + return { + text: this.submitButtonText, + attributes: { + variant: 'confirm', + disabled: this.submitDisabled, + loading: this.isLoading, + 'data-qa-selector': 'invite_button', + }, + }; + }, + actionCancel() { + return { + text: this.cancelButtonText, + }; + }, }, watch: { selectedAccessLevel: { @@ -151,7 +175,7 @@ export default { }, }, methods: { - reset() { + onReset() { // This component isn't necessarily disposed, // so we might need to reset it's state. this.selectedAccessLevel = this.defaultAccessLevel; @@ -159,14 +183,23 @@ export default { this.$emit('reset'); }, - closeModal() { - this.reset(); - this.$refs.modal.hide(); + onCloseModal(e) { + if (this.preventCancelDefault) { + e.preventDefault(); + } else { + this.onReset(); + this.$refs.modal.hide(); + } + + this.$emit('cancel'); }, changeSelectedItem(item) { this.selectedAccessLevel = item; }, - submit() { + onSubmit(e) { + // We never want to hide when submitting + e.preventDefault(); + this.$emit('submit', { accessLevel: this.selectedAccessLevel, expiresAt: this.selectedDate, @@ -192,9 +225,11 @@ export default { size="sm" :title="modalTitle" :header-close-label="$options.HEADER_CLOSE_LABEL" - @hidden="reset" - @close="reset" - @hide="reset" + :action-primary="actionPrimary" + :action-cancel="actionCancel" + @primary="onSubmit" + @cancel="onCloseModal" + @hidden="onReset" > - diff --git a/app/models/project.rb b/app/models/project.rb index 34f10befdba..91189afb72d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1484,7 +1484,7 @@ class Project < ApplicationRecord end def find_or_initialize_integration(name) - return if disabled_integrations.include?(name) + return if disabled_integrations.include?(name) || Integration.available_integration_names.exclude?(name) find_integration(integrations, name) || build_from_instance(name) || build_integration(name) end diff --git a/app/services/database/consistency_check_service.rb b/app/services/database/consistency_check_service.rb new file mode 100644 index 00000000000..e39bc8f25b8 --- /dev/null +++ b/app/services/database/consistency_check_service.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module Database + class ConsistencyCheckService + CURSOR_REDIS_KEY_TTL = 7.days + EMPTY_RESULT = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [] }.freeze + + def initialize(source_model:, target_model:, source_columns:, target_columns:) + @source_model = source_model + @target_model = target_model + @source_columns = source_columns + @target_columns = target_columns + @source_sort_column = source_columns.first + @target_sort_column = target_columns.first + end + + # This class takes two ActiveRecord models, and compares the selected columns + # of the two models tables, for the purposes of checking the consistency of + # mirroring of tables. For example Namespace and Ci::NamepaceMirror + # + # It compares up to 25 batches (1000 records / batch), or up to 30 seconds + # for all the batches in total. + # + # It saves the cursor of the next start_id (cusror) in Redis. If the start_id + # wasn't saved in Redis, for example, in the first run, it will choose some random start_id + # + # Example: + # service = Database::ConsistencyCheckService.new( + # source_model: Namespace, + # target_model: Ci::NamespaceMirror, + # source_columns: %w[id traversal_ids], + # target_columns: %w[namespace_id traversal_ids], + # ) + # result = service.execute + # + # result is a hash that has the following fields: + # - batches: Number of batches checked + # - matches: The number of matched records + # - mismatches: The number of mismatched records + # - mismatches_details: It's an array that contains details about the mismatched records. + # each record in this array is a hash of format {id: ID, source_table: [...], target_table: [...]} + # Each record represents the attributes of the records in the two tables. + # - start_id: The start id cursor of the current batch. means no records. + # - next_start_id: The ID that can be used for the next batch iteration check. means no records + def execute + start_id = next_start_id + + return EMPTY_RESULT if start_id.nil? + + result = consistency_checker.execute(start_id: start_id) + result[:start_id] = start_id + + save_next_start_id(result[:next_start_id]) + + result + end + + private + + attr_reader :source_model, :target_model, :source_columns, :target_columns, :source_sort_column, :target_sort_column + + def consistency_checker + @consistency_checker ||= Gitlab::Database::ConsistencyChecker.new( + source_model: source_model, + target_model: target_model, + source_columns: source_columns, + target_columns: target_columns + ) + end + + def next_start_id + return if min_id.nil? + + fetch_next_start_id || random_start_id + end + + # rubocop: disable CodeReuse/ActiveRecord + def min_id + @min_id ||= source_model.minimum(source_sort_column) + end + + def max_id + @max_id ||= source_model.minimum(source_sort_column) + end + # rubocop: enable CodeReuse/ActiveRecord + + def fetch_next_start_id + Gitlab::Redis::SharedState.with { |redis| redis.get(cursor_redis_shared_state_key)&.to_i } + end + + # This returns some random start_id, so that we don't always start checking + # from the start of the table, in case we lose the cursor in Redis. + def random_start_id + range_start = min_id + range_end = [min_id, max_id - Gitlab::Database::ConsistencyChecker::BATCH_SIZE].max + rand(range_start..range_end) + end + + def save_next_start_id(start_id) + Gitlab::Redis::SharedState.with do |redis| + redis.set(cursor_redis_shared_state_key, start_id, ex: CURSOR_REDIS_KEY_TTL) + end + end + + def cursor_redis_shared_state_key + "consistency_check_cursor:#{source_model.table_name}:#{target_model.table_name}" + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 86d36fd7f79..5e0de3376d7 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -327,6 +327,24 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:database_ci_namespace_mirrors_consistency_check + :worker_name: Database::CiNamespaceMirrorsConsistencyCheckWorker + :feature_category: :sharding + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: cronjob:database_ci_project_mirrors_consistency_check + :worker_name: Database::CiProjectMirrorsConsistencyCheckWorker + :feature_category: :sharding + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:database_drop_detached_partitions :worker_name: Database::DropDetachedPartitionsWorker :feature_category: :database diff --git a/app/workers/database/ci_namespace_mirrors_consistency_check_worker.rb b/app/workers/database/ci_namespace_mirrors_consistency_check_worker.rb new file mode 100644 index 00000000000..2b4253947ac --- /dev/null +++ b/app/workers/database/ci_namespace_mirrors_consistency_check_worker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Database + class CiNamespaceMirrorsConsistencyCheckWorker + include ApplicationWorker + include CronjobQueue # rubocop: disable Scalability/CronWorkerContext + + sidekiq_options retry: false + feature_category :sharding + data_consistency :sticky + idempotent! + + version 1 + + def perform + return if Feature.disabled?(:ci_namespace_mirrors_consistency_check, default_enabled: :yaml) + + results = ConsistencyCheckService.new( + source_model: Namespace, + target_model: Ci::NamespaceMirror, + source_columns: %w[id traversal_ids], + target_columns: %w[namespace_id traversal_ids] + ).execute + + log_extra_metadata_on_done(:results, results) + end + end +end diff --git a/app/workers/database/ci_project_mirrors_consistency_check_worker.rb b/app/workers/database/ci_project_mirrors_consistency_check_worker.rb new file mode 100644 index 00000000000..e9413256617 --- /dev/null +++ b/app/workers/database/ci_project_mirrors_consistency_check_worker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Database + class CiProjectMirrorsConsistencyCheckWorker + include ApplicationWorker + include CronjobQueue # rubocop: disable Scalability/CronWorkerContext + + sidekiq_options retry: false + feature_category :sharding + data_consistency :sticky + idempotent! + + version 1 + + def perform + return if Feature.disabled?(:ci_project_mirrors_consistency_check, default_enabled: :yaml) + + results = ConsistencyCheckService.new( + source_model: Project, + target_model: Ci::ProjectMirror, + source_columns: %w[id namespace_id], + target_columns: %w[project_id namespace_id] + ).execute + + log_extra_metadata_on_done(:results, results) + end + end +end diff --git a/config/feature_flags/development/ci_namespace_mirrors_consistency_check.yml b/config/feature_flags/development/ci_namespace_mirrors_consistency_check.yml new file mode 100644 index 00000000000..965fb904a15 --- /dev/null +++ b/config/feature_flags/development/ci_namespace_mirrors_consistency_check.yml @@ -0,0 +1,8 @@ +--- +name: ci_namespace_mirrors_consistency_check +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81836 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/356577 +milestone: '14.10' +type: development +group: group::sharding +default_enabled: false diff --git a/config/feature_flags/development/ci_project_mirrors_consistency_check.yml b/config/feature_flags/development/ci_project_mirrors_consistency_check.yml new file mode 100644 index 00000000000..9ba2865216d --- /dev/null +++ b/config/feature_flags/development/ci_project_mirrors_consistency_check.yml @@ -0,0 +1,8 @@ +--- +name: ci_project_mirrors_consistency_check +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81836 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/356583 +milestone: '14.10' +type: development +group: group::sharding +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 3c1bc98c34a..20983134889 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -763,6 +763,12 @@ Gitlab.ee do Settings.cron_jobs['loose_foreign_keys_cleanup_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['loose_foreign_keys_cleanup_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['loose_foreign_keys_cleanup_worker']['job_class'] = 'LooseForeignKeys::CleanupWorker' + Settings.cron_jobs['ci_namespace_mirrors_consistency_check_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['ci_namespace_mirrors_consistency_check_worker']['cron'] ||= '*/4 * * * *' + Settings.cron_jobs['ci_namespace_mirrors_consistency_check_worker']['job_class'] = 'Database::CiNamespaceMirrorsConsistencyCheckWorker' + Settings.cron_jobs['ci_project_mirrors_consistency_check_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['ci_project_mirrors_consistency_check_worker']['cron'] ||= '2-58/4 * * * *' + Settings.cron_jobs['ci_project_mirrors_consistency_check_worker']['job_class'] = 'Database::CiProjectMirrorsConsistencyCheckWorker' end # diff --git a/config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml b/config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml index 05603ec14e0..03a16b98a9b 100644 --- a/config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml +++ b/config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml @@ -167,6 +167,7 @@ options: - p_ci_templates_implicit_security_cluster_image_scanning - p_ci_templates_kaniko - p_ci_templates_qualys_iac_security + - p_ci_templates_database_liquibase distribution: - ce - ee diff --git a/config/metrics/counts_28d/20220307150912_p_ci_templates_database_liquibase_monthly.yml b/config/metrics/counts_28d/20220307150912_p_ci_templates_database_liquibase_monthly.yml new file mode 100644 index 00000000000..07a5548474a --- /dev/null +++ b/config/metrics/counts_28d/20220307150912_p_ci_templates_database_liquibase_monthly.yml @@ -0,0 +1,25 @@ +--- +key_path: redis_hll_counters.ci_templates.p_ci_templates_database_liquibase_monthly +description: "" +product_section: "" +product_stage: "" +product_group: "" +product_category: +value_type: number +status: active +milestone: "14.9" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81817 +time_frame: 28d +data_source: redis_hll +data_category: optional +instrumentation_class: RedisHLLMetric +distribution: +- ce +- ee +tier: +- free +- premium +- ultimate +options: + events: + - p_ci_templates_database_liquibase diff --git a/config/metrics/counts_7d/20210216184557_ci_templates_total_unique_counts_weekly.yml b/config/metrics/counts_7d/20210216184557_ci_templates_total_unique_counts_weekly.yml index 290ceb5754a..062ed14d649 100644 --- a/config/metrics/counts_7d/20210216184557_ci_templates_total_unique_counts_weekly.yml +++ b/config/metrics/counts_7d/20210216184557_ci_templates_total_unique_counts_weekly.yml @@ -167,6 +167,7 @@ options: - p_ci_templates_implicit_security_cluster_image_scanning - p_ci_templates_kaniko - p_ci_templates_qualys_iac_security + - p_ci_templates_database_liquibase distribution: - ce - ee diff --git a/config/metrics/counts_7d/20220307150854_p_ci_templates_database_liquibase_weekly.yml b/config/metrics/counts_7d/20220307150854_p_ci_templates_database_liquibase_weekly.yml new file mode 100644 index 00000000000..de9757b9f17 --- /dev/null +++ b/config/metrics/counts_7d/20220307150854_p_ci_templates_database_liquibase_weekly.yml @@ -0,0 +1,25 @@ +--- +key_path: redis_hll_counters.ci_templates.p_ci_templates_database_liquibase_weekly +description: "" +product_section: "" +product_stage: "" +product_group: "" +product_category: "" +value_type: number +status: active +milestone: "14.9" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81817 +time_frame: 7d +data_source: redis_hll +data_category: optional +instrumentation_class: RedisHLLMetric +distribution: +- ce +- ee +tier: +- free +- premium +- ultimate +options: + events: + - p_ci_templates_database_liquibase diff --git a/db/post_migrate/20220223124428_schedule_merge_topics_with_same_name.rb b/db/post_migrate/20220223124428_schedule_merge_topics_with_same_name.rb new file mode 100644 index 00000000000..7e79c89203a --- /dev/null +++ b/db/post_migrate/20220223124428_schedule_merge_topics_with_same_name.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class ScheduleMergeTopicsWithSameName < Gitlab::Database::Migration[1.0] + MIGRATION = 'MergeTopicsWithSameName' + BATCH_SIZE = 100 + + disable_ddl_transaction! + + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + def up + Topic.select('LOWER(name) as name').group('LOWER(name)').having('COUNT(*) > 1').order('LOWER(name)') + .in_groups_of(BATCH_SIZE, false).each_with_index do |group, i| + migrate_in((i + 1) * 2.minutes, MIGRATION, [group.map(&:name)]) + end + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20220223124428 b/db/schema_migrations/20220223124428 new file mode 100644 index 00000000000..b163accc957 --- /dev/null +++ b/db/schema_migrations/20220223124428 @@ -0,0 +1 @@ +8fb72b15bfaa1b58f87cb3f1836df1e8bfa1a5ddec4e480a2cb6a3c9fafe3bda \ No newline at end of file diff --git a/doc/api/project_level_variables.md b/doc/api/project_level_variables.md index 2251b0fc7fd..4205e6699fe 100644 --- a/doc/api/project_level_variables.md +++ b/doc/api/project_level_variables.md @@ -170,5 +170,5 @@ This parameter is used for filtering by attributes, such as `environment_scope`. Example usage: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/variables/VARIABLE_1?filter[environment_scope]=production" +curl --request DELETE --globoff --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/variables/VARIABLE_1?filter[environment_scope]=production" ``` diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb new file mode 100644 index 00000000000..07231098a5f --- /dev/null +++ b/lib/gitlab/background_migration/merge_topics_with_same_name.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to merge project topics with the same case insensitive name + class MergeTopicsWithSameName + # Temporary AR model for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + # Temporary AR model for project topic assignment + class ProjectTopic < ActiveRecord::Base + self.table_name = 'project_topics' + end + + def perform(topic_names) + topic_names.each do |topic_name| + topics = Topic.where('LOWER(name) = ?', topic_name) + .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) + .to_a + topic_to_keep = topics.shift + merge_topics(topic_to_keep, topics) if topics.any? + end + end + + private + + def merge_topics(topic_to_keep, topics_to_remove) + description = topic_to_keep.description + + topics_to_remove.each do |topic| + description ||= topic.description if topic.description.present? + process_avatar(topic_to_keep, topic) if topic.avatar.present? + + ProjectTopic.transaction do + ProjectTopic.where(topic_id: topic.id) + .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) + .update_all(topic_id: topic_to_keep.id) + ProjectTopic.where(topic_id: topic.id).delete_all + end + end + + Topic.where(id: topics_to_remove).delete_all + + topic_to_keep.update( + description: description, + total_projects_count: total_projects_count(topic_to_keep.id), + non_private_projects_count: non_private_projects_count(topic_to_keep.id) + ) + end + + # We intentionally use application code here because we need to copy/remove avatar files + def process_avatar(topic_to_keep, topic_to_remove) + topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) + topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) + unless topic_to_keep.avatar.present? + topic_to_keep.avatar = topic_to_remove.avatar + topic_to_keep.save! + end + + topic_to_remove.remove_avatar! + topic_to_remove.save! + end + + def total_projects_count(topic_id) + ProjectTopic.where(topic_id: topic_id).count + end + + def non_private_projects_count(topic_id) + ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') + .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count + end + end + end +end diff --git a/lib/gitlab/ci/templates/Database/liquibase.gitlab-ci.yml b/lib/gitlab/ci/templates/Database/liquibase.gitlab-ci.yml new file mode 100644 index 00000000000..49bafca9952 --- /dev/null +++ b/lib/gitlab/ci/templates/Database/liquibase.gitlab-ci.yml @@ -0,0 +1,149 @@ +# This file is a template, and might need editing before it works on your project. +# Here is a live project example that is using this template: +# https://gitlab.com/szandany/h2 + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Database/liquibase.gitlab-ci.yml + +# This template must be configured with CI/CD variables before it will work. +# See https://www.liquibase.com/blog/secure-database-developer-flow-using-gitlab-pipelines +# to learn how to configure the Liquibase template by using variables. +# Be sure to add the variables before running pipelines with this template. +# You may not want to run all the jobs in this template. You can comment out or delete the jobs you don't wish to use. + +# List of stages for jobs and their order of execution. +stages: + - build + - test + - deploy + - compare + + +# Helper functions to determine if the database is ready for deployments (function isUpToDate) or rollbacks (function isRollback) when tag is applied. +.functions: &functions | + function isUpToDate(){ + status=$(liquibase status --verbose) + if [[ $status == *'is up to date'* ]]; then + echo "database is already up to date" & exit 0 + fi; + } + + function isRollback(){ + if [ -z "$TAG" ]; then + echo "No TAG provided, running any pending changes" + elif [[ "$(liquibase rollbackSQL $TAG)" ]]; then + liquibase --logLevel=info --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log rollback $TAG && exit 0 + else exit 0 + fi; + } + + +# This is a series of Liquibase commands that can be run while doing database migrations from Liquibase docs at https://docs.liquibase.com/commands/home.html +.liquibase_job: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image at - https://hub.docker.com/r/liquibase/liquibase + before_script: + - liquibase --version + - *functions + - isRollback + - isUpToDate + - liquibase checks run + - liquibase update + - liquibase rollbackOneUpdate --force # This is a Pro command. Try Pro free trial here - https://liquibase.org/try-liquibase-pro-free + - liquibase tag $CI_PIPELINE_ID + - liquibase --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log --logLevel=info update + - liquibase history + artifacts: + paths: + - ${CI_JOB_NAME}_${CI_PIPELINE_ID}.log + expire_in: 1 week + + +# This job runs in the build stage, which runs first. +build-job: + extends: .liquibase_job + stage: build + environment: + name: DEV + script: + - echo "This job tested successfully with liquibase in DEV environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the test stage. It only starts when the job in the build stage completes successfully. +test-job: + extends: .liquibase_job + stage: test + environment: + name: TEST + script: + - echo "This job testsed successfully with liquibase in TEST environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the deploy stage. It only starts when the jobs in the test stage completes successfully. +deploy-prod: + extends: .liquibase_job + stage: deploy + environment: + name: PROD + script: + - echo "This job deployed successfully Liquibase in a production environment from the $CI_COMMIT_BRANCH branch." + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job compares dev database with test database to detect any drifts in the pipeline. Learn more about comparing database with Liquibase here https://docs.liquibase.com/commands/diff.html +DEV->TEST: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: TEST + script: + - echo "Comparing databases DEV --> TEST" + - liquibase diff + - liquibase --outputFile=diff_between_DEV_TEST.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_DEV_TEST.json + expire_in: 1 week + + +# This job compares test database with prod database to detect any drifts in the pipeline. +TEST->PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: PROD + script: + - echo "Comparing databases TEST --> PROD" + - liquibase diff + - liquibase --outputFile=diff_between_TEST_PROD.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_TEST_PROD.json + expire_in: 1 week + + +# This job creates a snapshot of prod database. You can use the snapshot file to run comparisons with the production database to investigate for any potential issues. https://www.liquibase.com/devsecops +snapshot PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: .post + environment: + name: PROD + script: + - echo "Snapshotting database PROD" + - liquibase --outputFile=snapshot_PROD_${CI_PIPELINE_ID}.json snapshot --snapshotFormat=json --log-level debug + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - snapshot_PROD_${CI_PIPELINE_ID}.json + expire_in: 1 week diff --git a/lib/gitlab/database/consistency_checker.rb b/lib/gitlab/database/consistency_checker.rb new file mode 100644 index 00000000000..e398fef744c --- /dev/null +++ b/lib/gitlab/database/consistency_checker.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConsistencyChecker + BATCH_SIZE = 1000 + MAX_BATCHES = 25 + MAX_RUNTIME = 30.seconds # must be less than the scheduling frequency of the ConsistencyCheck jobs + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize(source_model:, target_model:, source_columns:, target_columns:) + @source_model = source_model + @target_model = target_model + @source_columns = source_columns + @target_columns = target_columns + @source_sort_column = source_columns.first + @target_sort_column = target_columns.first + @result = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [] } + end + + # rubocop:disable Metrics/AbcSize + def execute(start_id:) + current_start_id = start_id + + return build_result(next_start_id: nil) if max_id.nil? + return build_result(next_start_id: min_id) if current_start_id > max_id + + @start_time = monotonic_time + + MAX_BATCHES.times do + if (current_start_id <= max_id) && !over_time_limit? + ids_range = current_start_id...(current_start_id + BATCH_SIZE) + # rubocop: disable CodeReuse/ActiveRecord + source_data = source_model.where(source_sort_column => ids_range) + .order(source_sort_column => :asc).pluck(*source_columns) + target_data = target_model.where(target_sort_column => ids_range) + .order(target_sort_column => :asc).pluck(*target_columns) + # rubocop: enable CodeReuse/ActiveRecord + + current_start_id += BATCH_SIZE + result[:matches] += append_mismatches_details(source_data, target_data) + result[:batches] += 1 + else + break + end + end + + result[:mismatches] = result[:mismatches_details].length + metrics_counter.increment({ source_table: source_model.table_name, result: "match" }, result[:matches]) + metrics_counter.increment({ source_table: source_model.table_name, result: "mismatch" }, result[:mismatches]) + + build_result(next_start_id: current_start_id > max_id ? min_id : current_start_id) + end + # rubocop:enable Metrics/AbcSize + + private + + attr_reader :source_model, :target_model, :source_columns, :target_columns, + :source_sort_column, :target_sort_column, :start_time, :result + + def build_result(next_start_id:) + { next_start_id: next_start_id }.merge(result) + end + + def over_time_limit? + (monotonic_time - start_time) >= MAX_RUNTIME + end + + # This where comparing the items happen, and building the diff log + # It returns the number of matching elements + def append_mismatches_details(source_data, target_data) + # Mapping difference the sort key to the item values + # source - target + source_diff_hash = (source_data - target_data).index_by { |item| item.shift } + # target - source + target_diff_hash = (target_data - source_data).index_by { |item| item.shift } + + matches = source_data.length - source_diff_hash.length + + # Items that exist in the first table + Different items + source_diff_hash.each do |id, values| + result[:mismatches_details] << { + id: id, + source_table: values, + target_table: target_diff_hash[id] + } + end + + # Only the items that exist in the target table + target_diff_hash.each do |id, values| + next if source_diff_hash[id] # It's already added + + result[:mismatches_details] << { + id: id, + source_table: source_diff_hash[id], + target_table: values + } + end + + matches + end + + # rubocop: disable CodeReuse/ActiveRecord + def min_id + @min_id ||= source_model.minimum(source_sort_column) + end + + def max_id + @max_id ||= source_model.maximum(source_sort_column) + end + # rubocop: enable CodeReuse/ActiveRecord + + def metrics_counter + @metrics_counter ||= Gitlab::Metrics.counter( + :consistency_checks, + "Consistency Check Results" + ) + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a39fa7aca4f..a5ec72fe8c9 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -615,3 +615,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_database_liquibase + category: ci_templates + redis_slot: ci_templates + aggregation: weekly diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 518c7407ba6..22ea2a620ef 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -93,6 +93,7 @@ describe QA::Support::Formatters::TestStatsFormatter do before do allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client } allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources } + allow_any_instance_of(RSpec::Core::Example::ExecutionResult).to receive(:run_time).and_return(0) # rubocop:disable RSpec/AnyInstanceOf end context "without influxdb variables configured" do diff --git a/spec/fixtures/avatars/avatar1.png b/spec/fixtures/avatars/avatar1.png new file mode 100644 index 0000000000000000000000000000000000000000..7e8afb39f173cf8d8602360f8e617f50b0605bbd GIT binary patch literal 1461 zcmb7^dpr{g6vyW=dF7caC{k8HDFx+y0?;(PZ&4vsW8r~t0N@E(YwJ8vp&`fsw3mYaH)?<< z2NXMcsi>e+06mK%Fl0nh4!N?qYlJ7ZG9}$q%rugGLQ((#o!8$|sUKe#tvE0CMZnP+ zVIo>WDgB7{$=V$CRg_%bY)+}-3)uUEL)~lFr@ojB(h|$>J-a~s@qPf9IuQLY9D3U6 zb`AA*`?mWq{CdXXE^)Kj7TYn+%OeSWZd}`KIhG-<>gw^9lr*_;@%(y!%GY;uurAnm zQMA!*yp}8#bq+$&D`i%L@63jt?sfkQ1@T2PM)qr2-L8EBf{#1wVNd(@O=665m=jx2`SES=-j^9AW1pKb8RGuC`MZK|;9B}TA>@`& z^OsSo^8|_S(qR>-Y0eORmFSxJtQ_LFx6}}d^<|~DDBa1Rli4N3s_2L6G9XM)t04oX z2|5sPq9-)&(X9t2F+wye;2`>H{~F6r0h>7r7KuX{+2#vLk^Ez<8*0QWhK}2X?s==1eCn6+`$Y_$(!Ce=str=C&oiv}<6E^Xc(G;#yG0M)0 z|0NeZxt^6y>%4d=#C=F66>l{?H4b=TO}JYqUdGe}#1Q9f0td?0X8hMQXi&Ki_UZQD z+3<#zSn}DiPmMy(;Sl<^4J7MJr1xE6x->ECmVexcwmHXbe1d8ET>il?YyKV+SD(q| zH8}u8R}Jssrb>j^PF#eNh)1q7R-bg1K&BsuduX@D0+}o4w|7okjaC?K6ktv?LSye2 zxg@~2DaxP}zueYX)oeKECsJz{Gi0X|_QEw=Egr4Ih>AX4d=nwN)n*-xrybr!c2>Iu^K|O^W9DwOnmRp<0`6SCxh=Q%!MU zMla$s4s;NXxrAUAK zbeT{#Y$0+=AgDrc*QGDDP?T4@xG6o~rzQbg7iPYYRF98V7)lKbipVVsAG8p5kYs6N z&8kLkh0bt=i8BRytqdv;HIHEAawKA^GT5$+uo1UlP7qWHt$8ji5SK2rHZG`yq);pHj zEymZaP(ZNRS6gY6Hnzins&7zB+j@wHCM*9mYKUCxouPHaSQ zD~hj{_9_Z7WWI8f7ap0mrF>K-_9A^$YFJo9H-ZgxE*JMY*-qo@Ha)60i{5zzz85J-+?GpWT#kiay z2hBgaN_u6eo|-oD=g{RXf} zLDb0WuOsa#2AiM&(n3eW+PLEO{`EK=`a}hn+3a4Gr#^IR3)Ju~SY0mMQW*MpFQ=KS zKdUvn?(UVF)oeQso?q3h~mzK z_)EcZev`)Y&#i&H8*J{yQ>lVT&y~WsOQTRCuUCX#d0qPecDi#yt^;IIF6bwS4mG~r z?Y|i7O3nn6cpv(g;;;zPt85Ni$Ej4{M&qF7Sc-bGa)(K{`oW;9w#(z=U;ga12>M7l zjM2+Do9Q61JmEK0ynn4nxpG{@s|LToT5@#XG_(bvZ&B=1IBK2HXcfyO_zebaSw#Vp z6SJr>MuP&poL z@W31tV4CgTMIMAG*#6NXq0jOx>!Lh$yvLQ#a`63(yx=4zFDeH1umWGUHudALsZ%;( z3Eh&Z`iHNJfU+eAHU7_AHL`q{6_e8N791UE!Jh^KqxE2)UKl*Ik!=>GONMV}D`hxt znh{GjF@p`O@pWVCpRPmFYP%LVYT(5Q5KeS%cwwzi;aH1WBt`wC~%xG3M9$(Hv{ANvovHD{-Q*r|d)zx*9 zeiINdimF+b$D|0dAQK5to4&N+oY6CY!x;4c{vvFq-sg7KIkSGGo(eEjqCxlj+*Pc zO+O37zPrCHRBHm##Ck-tXH!)r1w0bz5Wm=a)cN#w%dZgjBWHDSM7|w@$4d| zF$ivN81wsc>yB-MY-!#B`)H@uZHXx;C!AvN*Mi!#nxTz{{F!o8WkParztLXzres0< z0q0#5*H;>&hYOBeNOcsIz?0^vdy$h47qtAuSVMnWq)|o-QU8SWZNBRQo>2aH(eAge z)|rGd*r=kQ8`3>sQWml1|Go6Dts58nYH^HwSkK?NE*>l=7qqiSqRP}%*|FgY9wxrj zvofv+>6Rpx@t)X60dOG>Mk@^LAA&WVE zq9{&hNa3yT@}u!Jd!1zTEM^p`EiAR=GMs*o;CQ9;VsgU=2(}wZw6#K!+59W|4I6hg z0w3e29EX_-$GFN4X+w?Pg8#}xfaOQ4#+t8R&NPbbkX=X6ZQit|G0gu!7R!}i%GDwh zjo$~iM;Ie6ZR>YMCB9eKo|zHOFi_P6e!!c}>@@2W69e$lUizFC+|gRDjnRT4=pMH7Ulcjo;8@63G9XTEdh{QCW-*~6?re82Gl000ow8sc!y-e2MY zp6_X19Q!%gLLIDb0$L_7QO*sAAJ`5I0JLTCpZvjj?)B_#?^&Gl|D8%s%dztx;U^9- zM}V-L&JBAnWPBEZ*hgO8R`?B*etzRcTDyh<0D?om#5Rhk^92C7L%@$)jhL zIy}iLPFubBv$IDZ=Ke}-ezwD269p^H3Z1jg0VF6M=Z*R5;%$}x1@6M6M|y<@7Rw*U z&_=Gml(yWC<2{|b_Cfa~5iwFQ@hbbMgv6R5AXS-pBxVPE<&Bojp30IWE55ypvwm%t zu4Gyw_5lZ$)4C;^Y@PhbRpJNiVlotCe`qo$U=2Prfqq42q|?w;ov#u3o9-Sy8iREk zD+@cGnhTVzFt$GLF;psxyez~s)fP){j9B_M@>?o z7;+Nfc1RcA;`5*lGqy@=0{PcL)-J%+-z)?VsD<`50A+2Ya ze6n*)XJ7FxXMZzOGKSbVbgJZUW*vZ+eVEllbXj9l?c>SPP2H4%51FqI5IXyvWAkEB z5;r=?hnaeMQ;CI{jS`-Q6WMD_^y!2H4d>$KfIa|9KLw6tn(-cUDHMJ)G@!56z}V zW=$x3JoLbrXIe&>3!f}R@e|5b@&3p@y0aS?4HVJ5I)R+=JqQ9u7!^{H0`T@+x1?b= zTRnlHzNc3Eavufj@T>wM&Jg~n?Cs1<8Mx@MZ7BEJ)hSKg-Cdg=fVHWhdnp!e{6;G! zh>@6*i8gM~8hwYzjGJZr*;*>K(AI}}3F zvEh(v$_;F%q#KK!GPypFRN{ZRH8YO(7?nWzmWNp!iI^>cQ$ZcnnX`M&Os)6x0`R$S zpW2UY>dss!2PkvLv^DFdeTn~sS++mKzKp_;(zduigwT<-Uc@_1UO%eF6*eFdu%C`#ROwr zGA$u~PbVWXh9sbG|GNr!P4umFn4OB$VfCqS&?_zdlwpyY)twk!e4*14sIy0$8BBZ8 zX~Q)Ww^GHc=PrJ?R`<&sPW(#Rj_IUKQ2$YWET}0$WSJg4pkJa&Y5T?-@-4e77Mp$f zap?}oFH-UjGleh2S8^N3<&O7|a9V7D`%Q||`i) zG5`pJ1y&+~iQa}ln7AKkvk4=St7hVJ7nG$$qV__K7jqUS67R)YZE!U~Vy(7BO7v^A z(;f50YKY$ZdJ;>iG5W+96PB{rlF1fK@czQJxsphugwrI1#Pv@g32{Fq-zd723y=0f z!N`CmW9*Pz)Lpsvl=X#UG(hDQS ziN6}SzT)RH!M~1MmUS;@hP3Zz$gNdas?2nr@ y|MXd&gKIO`(Ovlub_py(u}JIy literal 0 HcmV?d00001 diff --git a/spec/fixtures/avatars/avatar4.png b/spec/fixtures/avatars/avatar4.png new file mode 100644 index 0000000000000000000000000000000000000000..647ee193cbd82d92eaa06a191c5e9d46a03d836b GIT binary patch literal 1624 zcmb7_YgEz)7{>pSW?H5@#%v}^VzV<7G<7ab5sr$a65eL&+-lv%aw_75(2O)A%}LXV zmrBE17^$f_9imn^%$m3HhcpuBl|+dW@otXxVc++j^E|)vp7%WO$M?-95^zR_=7s5CZ^>%$*yj^t8PbF(A}O%l})I zdcM8dj7efJAp|giJ9rUK#?zVQ4P66cDuj7Pj&^1c?-xMrgoB;s6e7rX% zl=`jm&MYevzAH<=NP=W_Zi({pf(DWKQ;_~6X=rz71Ih2HwL0YB9>XjwoA@tma%&Yl zlH&QG{uV@Zo0l4=6bU?IM{C(A3#ce@^O%MuoxaO_$W5D(z?2376r*5IipStK=h4j2 ztkWoH^c|>3V{`pQN9E^Dr;vx3BMIztO?N@h{Jtx|D_qkXSD5$uvy##P974%ey}9tC zi_r{r*@uijvFpxB!sJ&$(#9+M(kG7f^;G-P_{R-ZXxzKflHn&k-^}`wug`hR__k*I zSUE0*Zbat$TiL56@Bc8arc`xGz2_S&$y2SQZe#DgRB>-Nx;%dF-X7zWGTk9z4;I19 z4uhpl=;W>s6@Ydgqvl?>D;C^cd99^RF2yX?+j<*P+k3ih*_21}3kIsYMH{07);}0EHcGHQQ1!HkKmzJvA@nUrms1Y4+pd1d?#JnWUV)Ysj zh+5hb+&`@{%N5<)78Ki9xqAr}o$U3XY}RefQEXAQ$+)?7m+#1O3$u?{n(r;*$`2f> zlcFLdE0#8Kg)S|X^<)ulsrgWKL9ej z_WOGDyiYp&G}yYh5td84HW{9ny{mmngVyinEZ-Wzgu_tjPWDlBkRj)qM(_VV+ts;K zr`C8Nyfk302H-$%uD!ck`TL8BMYXfF-zDl`&{d0Yvw>iYLU6udS}`iCsa9#Ti~w@s zWR2JBMM{DuQOC+PgM!5<%0VIWTIYc9NQYSqQPGjyX&uiH3&3DU z<4PHz-BBzER|U3pg7<30&6K)U-v!>NS#{Z~VMc_U+PKh%d|1g%5tOC0KlV}x?9Rw? z_%{mCVe5vQq^cTb!R8cVrWW<5vg@z<> zeN*nvPb~EHINisB9xmX>V~D5g7HosMUR6KZX7|P>$jUOI7$Xm8@Avcy%p0^#>I=7$ zk$hYhaPY#X?c%Y4hZbAzCK`*@o_U2eTye3z9k_Mq99(fE7XA28eixlMKuL`)Sv9&! zlLW5iY_Q5lG*xwk?@UJ^T{7(QHwrP;%!IwkaB0hx^`A9R!mAm)< z?FOIZiYb|!x|~Q3SFePYJ=>U711Xukf(gm`b5Y;16I|?Z&>F*`^Yvm|F$%XJ6JYC? zOV7Xl`hWos_9IQV+1`^p@Wrc4ryJ_sp&f9n=&UV$lJX_p30zpslf->^22F9WJ$W|A zU1~O6HATJ@R<_VPzGiZ-93VX3o^f=^eieVv?|pB9B(qWYDfTac0dZW&mKF>=l~N#x zkzcGGySqkHO*6lm&>uTKlr#x*EP=-hP0`PocYaGmJc*tlQnyy5&{Jphtl($HpLD<{r3}3jkqdE0aG;uXXH9B5pDlkp6v)=t6 zjqQ=tnl3hdpo}Y0k7qSFgo&Ls==E~iel!;Gfy2Nr5*kF0`NWr?|SUX GjDG;T_ZI;G literal 0 HcmV?d00001 diff --git a/spec/fixtures/avatars/avatar5.png b/spec/fixtures/avatars/avatar5.png new file mode 100644 index 0000000000000000000000000000000000000000..27e973dc5e3855fa03c35deb8c9d232b948b5335 GIT binary patch literal 1700 zcmb7_`8OMg7RMu`i6s(Rl0hf!puEQn)t*{{h=xQhjipj8qGL%_?G$P4he~a+6eGsI z6@!=-P0?z!8e3{Bt&-~0E{J_8Ui!}a2i`sBdq4Mk@43I;OC#Fgp%Mxb0001LLBQDw z+5STiQDGhwPXP)c5^RSz1=J5HEeMSm1#6830GR2L-*170{sPg;!A!{iH_AZf9N`W; z#LmVZ06(K`LUap=NiS-6$>psoE5tn#&cp)!u9%d$P7&?7cQqQK}jf@gKqa_2msm$f_LwO@6~+TwPnECU0JYwFXv4XDn#A zU`cpYAAA$OXME__(u}iy(L;m2rC+=cOc5H2>f;^oH&Ln+2nCj>-j$HL4tFWV=%%#7 zS~EoBqv}!IMtSA^n^`XNu`K8L?dDS?YQd$mBRpia7u#SL`NWIuXykM?o%#F46eF+9 zXq|k|DZpE97NLxJQz4B6R@|u7e#=GnDJ8qT<@qg6!Mxj8;p=eGZBDsDgib#-kA3a@ z3PbG)$(;HW*gVsDpGs}kg-&W~UV+vYPrt>gu8EzUup!(MldQ6j z&hj5guVR3>&mApVXbGgxGoMTpzEKnH z%mStEMOo^?JJo$XZIRBCuVqI~NYPj?qr9dwW5FT+?u(&sbSlI$rPQBBmGgU96 zmuwMzezN*Ig+;t$L=TW;IY^YroO^b34whdWP@pz53hp{Bv6pG^DT;c7Jgjz<)a`4A zcvP}PIb~_Mpd!h~=qp=ku5uyCr^8L>%Wu3nQ9v|UE9emnnV`JSFx*RIx}mUqUy9S~ zxZ}`n>OKx+>jPemHl%4)( zI?hkKY{FUwff^}S^MM%hbY`dosNVt2=8|>`^{a?P*XbGX>F-y= zBohLWi2{kacG2&}@h|v)K3uP9-|bDpJb&}o@n_rHhWPUsk0B6%tcFeH5+Sz^Eu1$v zZXwC`hBfTrDNrH1b)kn%Ww8-ofOH=t*NGsYX#TtB9LEacu9RK`#4XZx)Ot8HH)_D) zu)=p~y`S*vDfiE!`2h{TLrd8hg*Ip2(!heebSgnzvmo(;X{d3#4u2S#+1&bpvKe{p zo*djRE?Bvew)H|55&?9fN=K4^A{m;DmuPSYYdAZtPpk?9_vD%e2jFV6oD1^N@s;b@ zA#55WXld`)2mG*32qj1j*7Go10BTF!lf$!w{bWo}#p^_6mk5KomRr8Sj)~=rgNKYe zl)I*t7uLEsIVkPFOe>XL1Y)_&z}Z-f_kp^fk|vZhkJ^Q$6um)Ha5=5N@wC!da)R>2 znTjIcuvI~lzt)7bwD`*-JN=a@R5t!TZ5ca_h<*b;sOnuQ%5Ax*YMEHPw)&}>Y|ThK zOt~=yJGEjT!;)vDUPBxH-g9xg|5lgCDigAkWHR(ILQkufrI|c-J$6|mSDbPpZ+mr1 zAo5Ms<*3g?G|50|sU$bVQ-&gfY?=E&8K(j2Frn$+QO1#w@ZBV5J%Tfu`LnT}*>44V zFfyv2MaV4i2Bx`mC;bE4*Fph8iS)oN^tRk2_|?y`z5Yrs!Mx$x zm}8o}`DB26ru+!laOYygms$pz`891o{Q3#YIi;u4gEZwx&|pIsz2nh5DJ93`WpTvv z&9yA?5N@65x#z~WQLTv00N~{126^BIpN?lA(J48_Wc}qbyI@xiYySI@= z@k^`Ohxsg=U5kjObcZ-w!QuvFj_eiL+vusI^nZ51uG{$L$P#S#Dx9~SQ2ID`VC>R^ zf<1!j6~OS!Lo4#X6dm^rp?rol8iO5`=HWKkIT-b&ZJp(*cZK!tJV%WGZlR%8PwBg3 kt+JE { wrapper = null; }); + const findModal = () => wrapper.findComponent(GlModal); const findGroupSelect = () => wrapper.findComponent(GroupSelect); const findIntroText = () => wrapper.findByTestId('modal-base-intro-text').text(); - const findCancelButton = () => wrapper.findByTestId('cancel-button'); - const findInviteButton = () => wrapper.findByTestId('invite-button'); const findMembersFormGroup = () => wrapper.findByTestId('members-form-group'); const membersFormGroupInvalidFeedback = () => findMembersFormGroup().attributes('invalid-feedback'); - const clickInviteButton = () => findInviteButton().vm.$emit('click'); - const clickCancelButton = () => findCancelButton().vm.$emit('click'); - const triggerGroupSelect = (val) => findGroupSelect().vm.$emit('input', val); const findBase = () => wrapper.findComponent(InviteModalBase); - const hideModal = () => wrapper.findComponent(GlModal).vm.$emit('hide'); + const triggerGroupSelect = (val) => findGroupSelect().vm.$emit('input', val); + const emitEventFromModal = (eventName) => () => + findModal().vm.$emit(eventName, { preventDefault: jest.fn() }); + const hideModal = emitEventFromModal('hidden'); + const clickInviteButton = emitEventFromModal('primary'); + const clickCancelButton = emitEventFromModal('cancel'); describe('displaying the correct introText and form group description', () => { describe('when inviting to a project', () => { diff --git a/spec/frontend/invite_members/components/invite_members_modal_spec.js b/spec/frontend/invite_members/components/invite_members_modal_spec.js index dd16bb48cb8..1bf44fbed71 100644 --- a/spec/frontend/invite_members/components/invite_members_modal_spec.js +++ b/spec/frontend/invite_members/components/invite_members_modal_spec.js @@ -85,12 +85,13 @@ describe('InviteMembersModal', () => { mock.restore(); }); + const findModal = () => wrapper.findComponent(GlModal); const findBase = () => wrapper.findComponent(InviteModalBase); const findIntroText = () => wrapper.findByTestId('modal-base-intro-text').text(); - const findCancelButton = () => wrapper.findByTestId('cancel-button'); - const findInviteButton = () => wrapper.findByTestId('invite-button'); - const clickInviteButton = () => findInviteButton().vm.$emit('click'); - const clickCancelButton = () => findCancelButton().vm.$emit('click'); + const emitEventFromModal = (eventName) => () => + findModal().vm.$emit(eventName, { preventDefault: jest.fn() }); + const clickInviteButton = emitEventFromModal('primary'); + const clickCancelButton = emitEventFromModal('cancel'); const findMembersFormGroup = () => wrapper.findByTestId('members-form-group'); const membersFormGroupInvalidFeedback = () => findMembersFormGroup().attributes('invalid-feedback'); @@ -276,7 +277,7 @@ describe('InviteMembersModal', () => { }); it('renders the modal with the correct title', () => { - expect(wrapper.findComponent(GlModal).props('title')).toBe(MEMBERS_MODAL_CELEBRATE_TITLE); + expect(findModal().props('title')).toBe(MEMBERS_MODAL_CELEBRATE_TITLE); }); it('includes the correct celebration text and emoji', () => { @@ -337,7 +338,7 @@ describe('InviteMembersModal', () => { }); it('sets isLoading on the Invite button when it is clicked', () => { - expect(findInviteButton().props('loading')).toBe(true); + expect(findModal().props('actionPrimary').attributes.loading).toBe(true); }); it('calls Api addGroupMembersByUserId with the correct params', () => { @@ -380,7 +381,7 @@ describe('InviteMembersModal', () => { expect(membersFormGroupInvalidFeedback()).toBe('Member already exists'); expect(findMembersSelect().props('validationState')).toBe(false); - expect(findInviteButton().props('loading')).toBe(false); + expect(findModal().props('actionPrimary').attributes.loading).toBe(false); }); describe('clearing the invalid state and message', () => { @@ -414,7 +415,7 @@ describe('InviteMembersModal', () => { }); it('clears the error when the modal is hidden', async () => { - wrapper.findComponent(GlModal).vm.$emit('hide'); + findModal().vm.$emit('hidden'); await nextTick(); @@ -432,7 +433,7 @@ describe('InviteMembersModal', () => { expect(membersFormGroupInvalidFeedback()).toBe('Member already exists'); expect(findMembersSelect().props('validationState')).toBe(false); - expect(findInviteButton().props('loading')).toBe(false); + expect(findModal().props('actionPrimary').attributes.loading).toBe(false); findMembersSelect().vm.$emit('clear'); @@ -440,7 +441,7 @@ describe('InviteMembersModal', () => { expect(membersFormGroupInvalidFeedback()).toBe(''); expect(findMembersSelect().props('validationState')).toBe(null); - expect(findInviteButton().props('loading')).toBe(false); + expect(findModal().props('actionPrimary').attributes.loading).toBe(false); }); it('displays the generic error for http server error', async () => { @@ -542,7 +543,7 @@ describe('InviteMembersModal', () => { expect(membersFormGroupInvalidFeedback()).toBe(expectedSyntaxError); expect(findMembersSelect().props('validationState')).toBe(false); - expect(findInviteButton().props('loading')).toBe(false); + expect(findModal().props('actionPrimary').attributes.loading).toBe(false); }); it('displays the restricted email error when restricted email is invited', async () => { @@ -554,7 +555,7 @@ describe('InviteMembersModal', () => { expect(membersFormGroupInvalidFeedback()).toContain(expectedEmailRestrictedError); expect(findMembersSelect().props('validationState')).toBe(false); - expect(findInviteButton().props('loading')).toBe(false); + expect(findModal().props('actionPrimary').attributes.loading).toBe(false); }); it('displays the successful toast message when email has already been invited', async () => { diff --git a/spec/frontend/invite_members/components/invite_modal_base_spec.js b/spec/frontend/invite_members/components/invite_modal_base_spec.js index 9e17112fb15..8355ae67f20 100644 --- a/spec/frontend/invite_members/components/invite_modal_base_spec.js +++ b/spec/frontend/invite_members/components/invite_modal_base_spec.js @@ -49,8 +49,6 @@ describe('InviteModalBase', () => { const findDatepicker = () => wrapper.findComponent(GlDatepicker); const findLink = () => wrapper.findComponent(GlLink); const findIntroText = () => wrapper.findByTestId('modal-base-intro-text').text(); - const findCancelButton = () => wrapper.findByTestId('cancel-button'); - const findInviteButton = () => wrapper.findByTestId('invite-button'); const findMembersFormGroup = () => wrapper.findByTestId('members-form-group'); describe('rendering the modal', () => { @@ -67,15 +65,21 @@ describe('InviteModalBase', () => { }); it('renders the Cancel button text correctly', () => { - expect(findCancelButton().text()).toBe(CANCEL_BUTTON_TEXT); + expect(wrapper.findComponent(GlModal).props('actionCancel')).toMatchObject({ + text: CANCEL_BUTTON_TEXT, + }); }); - it('renders the Invite button text correctly', () => { - expect(findInviteButton().text()).toBe(INVITE_BUTTON_TEXT); - }); - - it('renders the Invite button modal without isLoading', () => { - expect(findInviteButton().props('loading')).toBe(false); + it('renders the Invite button correctly', () => { + expect(wrapper.findComponent(GlModal).props('actionPrimary')).toMatchObject({ + text: INVITE_BUTTON_TEXT, + attributes: { + variant: 'confirm', + disabled: false, + loading: false, + 'data-qa-selector': 'invite_button', + }, + }); }); describe('rendering the access levels dropdown', () => { @@ -114,7 +118,7 @@ describe('InviteModalBase', () => { isLoading: true, }); - expect(findInviteButton().props('loading')).toBe(true); + expect(wrapper.findComponent(GlModal).props('actionPrimary').attributes.loading).toBe(true); }); it('with invalidFeedbackMessage, set members form group validation state', () => { diff --git a/spec/helpers/application_settings_helper_spec.rb b/spec/helpers/application_settings_helper_spec.rb index 26d48bef24e..c93762416f5 100644 --- a/spec/helpers/application_settings_helper_spec.rb +++ b/spec/helpers/application_settings_helper_spec.rb @@ -149,7 +149,7 @@ RSpec.describe ApplicationSettingsHelper do end end - describe '.storage_weights' do + describe '#storage_weights' do let(:application_setting) { build(:application_setting) } before do @@ -158,12 +158,13 @@ RSpec.describe ApplicationSettingsHelper do stub_application_setting(repository_storages_weighted: { 'default' => 100, 'storage_1' => 50, 'storage_2' => nil }) end - it 'returns storages correctly' do - expect(helper.storage_weights).to eq(OpenStruct.new( - default: 100, - storage_1: 50, - storage_2: 0 - )) + it 'returns storage objects with assigned weights' do + expect(helper.storage_weights) + .to have_attributes( + default: 100, + storage_1: 50, + storage_2: 0 + ) end end diff --git a/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb b/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb new file mode 100644 index 00000000000..254b4fea698 --- /dev/null +++ b/spec/lib/gitlab/background_migration/merge_topics_with_same_name_spec.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MergeTopicsWithSameName, schema: 20220223124428 do + def set_avatar(topic_id, avatar) + topic = ::Projects::Topic.find(topic_id) + topic.avatar = avatar + topic.save! + topic.avatar.absolute_path + end + + it 'merges project topics with same case insensitive name' do + namespaces = table(:namespaces) + projects = table(:projects) + topics = table(:topics) + project_topics = table(:project_topics) + + group = namespaces.create!(name: 'group', path: 'group') + project_1 = projects.create!(namespace_id: group.id, visibility_level: 20) + project_2 = projects.create!(namespace_id: group.id, visibility_level: 10) + project_3 = projects.create!(namespace_id: group.id, visibility_level: 0) + topic_1_keep = topics.create!( + name: 'topic1', + description: 'description 1 to keep', + total_projects_count: 2, + non_private_projects_count: 2 + ) + topic_1_remove = topics.create!( + name: 'TOPIC1', + description: 'description 1 to remove', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_2_remove = topics.create!( + name: 'topic2', + total_projects_count: 0 + ) + topic_2_keep = topics.create!( + name: 'TOPIC2', + description: 'description 2 to keep', + total_projects_count: 1 + ) + topic_3_remove_1 = topics.create!( + name: 'topic3', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_3_keep = topics.create!( + name: 'Topic3', + total_projects_count: 2, + non_private_projects_count: 2 + ) + topic_3_remove_2 = topics.create!( + name: 'TOPIC3', + description: 'description 3 to keep', + total_projects_count: 2, + non_private_projects_count: 1 + ) + topic_4_keep = topics.create!( + name: 'topic4' + ) + + project_topics_1 = [] + project_topics_3 = [] + project_topics_removed = [] + + project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_1.id) + project_topics_1 << project_topics.create!(topic_id: topic_1_keep.id, project_id: project_2.id) + project_topics_removed << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_2.id) + project_topics_1 << project_topics.create!(topic_id: topic_1_remove.id, project_id: project_3.id) + + project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_1.id) + project_topics_3 << project_topics.create!(topic_id: topic_3_keep.id, project_id: project_2.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_1.id) + project_topics_3 << project_topics.create!(topic_id: topic_3_remove_1.id, project_id: project_3.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_1.id) + project_topics_removed << project_topics.create!(topic_id: topic_3_remove_2.id, project_id: project_3.id) + + avatar_paths = { + topic_1_keep: set_avatar(topic_1_keep.id, fixture_file_upload('spec/fixtures/avatars/avatar1.png')), + topic_1_remove: set_avatar(topic_1_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar2.png')), + topic_2_remove: set_avatar(topic_2_remove.id, fixture_file_upload('spec/fixtures/avatars/avatar3.png')), + topic_3_remove_1: set_avatar(topic_3_remove_1.id, fixture_file_upload('spec/fixtures/avatars/avatar4.png')), + topic_3_remove_2: set_avatar(topic_3_remove_2.id, fixture_file_upload('spec/fixtures/avatars/avatar5.png')) + } + + subject.perform(%w[topic1 topic2 topic3 topic4]) + + # Topics + [topic_1_keep, topic_2_keep, topic_3_keep, topic_4_keep].each(&:reload) + expect(topic_1_keep.name).to eq('topic1') + expect(topic_1_keep.description).to eq('description 1 to keep') + expect(topic_1_keep.total_projects_count).to eq(3) + expect(topic_1_keep.non_private_projects_count).to eq(2) + expect(topic_2_keep.name).to eq('TOPIC2') + expect(topic_2_keep.description).to eq('description 2 to keep') + expect(topic_2_keep.total_projects_count).to eq(0) + expect(topic_2_keep.non_private_projects_count).to eq(0) + expect(topic_3_keep.name).to eq('Topic3') + expect(topic_3_keep.description).to eq('description 3 to keep') + expect(topic_3_keep.total_projects_count).to eq(3) + expect(topic_3_keep.non_private_projects_count).to eq(2) + expect(topic_4_keep.reload.name).to eq('topic4') + + [topic_1_remove, topic_2_remove, topic_3_remove_1, topic_3_remove_2].each do |topic| + expect { topic.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + # Topic avatars + expect(topic_1_keep.avatar).to eq('avatar1.png') + expect(File.exist?(::Projects::Topic.find(topic_1_keep.id).avatar.absolute_path)).to be_truthy + expect(topic_2_keep.avatar).to eq('avatar3.png') + expect(File.exist?(::Projects::Topic.find(topic_2_keep.id).avatar.absolute_path)).to be_truthy + expect(topic_3_keep.avatar).to eq('avatar4.png') + expect(File.exist?(::Projects::Topic.find(topic_3_keep.id).avatar.absolute_path)).to be_truthy + + [:topic_1_remove, :topic_2_remove, :topic_3_remove_1, :topic_3_remove_2].each do |topic| + expect(File.exist?(avatar_paths[topic])).to be_falsey + end + + # Project Topic assignments + project_topics_1.each do |project_topic| + expect(project_topic.reload.topic_id).to eq(topic_1_keep.id) + end + + project_topics_3.each do |project_topic| + expect(project_topic.reload.topic_id).to eq(topic_3_keep.id) + end + + project_topics_removed.each do |project_topic| + expect { project_topic.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end +end diff --git a/spec/lib/gitlab/database/consistency_checker_spec.rb b/spec/lib/gitlab/database/consistency_checker_spec.rb new file mode 100644 index 00000000000..2ff79d20786 --- /dev/null +++ b/spec/lib/gitlab/database/consistency_checker_spec.rb @@ -0,0 +1,189 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::ConsistencyChecker do + let(:batch_size) { 10 } + let(:max_batches) { 4 } + let(:max_runtime) { described_class::MAX_RUNTIME } + let(:metrics_counter) { Gitlab::Metrics.registry.get(:consistency_checks) } + + subject(:consistency_checker) do + described_class.new( + source_model: Namespace, + target_model: Ci::NamespaceMirror, + source_columns: %w[id traversal_ids], + target_columns: %w[namespace_id traversal_ids] + ) + end + + before do + stub_const("#{described_class.name}::BATCH_SIZE", batch_size) + stub_const("#{described_class.name}::MAX_BATCHES", max_batches) + redis_shared_state_cleanup! # For Prometheus Counters + end + + after do + Gitlab::Metrics.reset_registry! + end + + describe '#over_time_limit?' do + before do + allow(consistency_checker).to receive(:start_time).and_return(0) + end + + it 'returns true only if the running time has exceeded MAX_RUNTIME' do + allow(consistency_checker).to receive(:monotonic_time).and_return(0, max_runtime - 1, max_runtime + 1) + expect(consistency_checker.monotonic_time).to eq(0) + expect(consistency_checker.send(:over_time_limit?)).to eq(false) + expect(consistency_checker.send(:over_time_limit?)).to eq(true) + end + end + + describe '#execute' do + context 'when empty tables' do + it 'returns an empty response' do + expected_result = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [], next_start_id: nil } + expect(consistency_checker.execute(start_id: 1)).to eq(expected_result) + end + end + + context 'when the tables contain matching items' do + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + end + + it 'does not process more than MAX_BATCHES' do + max_batches = 3 + stub_const("#{described_class.name}::MAX_BATCHES", max_batches) + result = consistency_checker.execute(start_id: Namespace.minimum(:id)) + expect(result[:batches]).to eq(max_batches) + expect(result[:matches]).to eq(max_batches * batch_size) + end + + it 'doesn not exceed the MAX_RUNTIME' do + allow(consistency_checker).to receive(:monotonic_time).and_return(0, max_runtime - 1, max_runtime + 1) + result = consistency_checker.execute(start_id: Namespace.minimum(:id)) + expect(result[:batches]).to eq(1) + expect(result[:matches]).to eq(1 * batch_size) + end + + it 'returns the correct number of matches and batches checked' do + expected_result = { + next_start_id: Namespace.minimum(:id) + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: max_batches, + matches: max_batches * batch_size, + mismatches: 0, + mismatches_details: [] + } + expect(consistency_checker.execute(start_id: Namespace.minimum(:id))).to eq(expected_result) + end + + it 'returns the min_id as the next_start_id if the check reaches the last element' do + expect(Gitlab::Metrics).to receive(:counter).at_most(:once) + .with(:consistency_checks, "Consistency Check Results") + .and_call_original + + # Starting from the 5th last element + start_id = Namespace.all.order(id: :desc).limit(5).pluck(:id).last + expected_result = { + next_start_id: Namespace.first.id, + batches: 1, + matches: 5, + mismatches: 0, + mismatches_details: [] + } + expect(consistency_checker.execute(start_id: start_id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(0) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(5) + end + end + + context 'when some items are missing from the first table' do + let(:missing_namespace) { Namespace.all.order(:id).limit(2).last } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + missing_namespace.delete + end + + it 'reports the missing elements' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: max_batches, + matches: 39, + mismatches: 1, + mismatches_details: [{ + id: missing_namespace.id, + source_table: nil, + target_table: [missing_namespace.traversal_ids] + }] + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(1) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(39) + end + end + + context 'when some items are missing from the second table' do + let(:missing_ci_namespace_mirror) { Ci::NamespaceMirror.all.order(:id).limit(2).last } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + missing_ci_namespace_mirror.delete + end + + it 'reports the missing elements' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: 4, + matches: 39, + mismatches: 1, + mismatches_details: [{ + id: missing_ci_namespace_mirror.namespace_id, + source_table: [missing_ci_namespace_mirror.traversal_ids], + target_table: nil + }] + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(1) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(39) + end + end + + context 'when elements are different between the two tables' do + let(:different_namespaces) { Namespace.order(:id).limit(max_batches * batch_size).sample(3).sort_by(&:id) } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + + different_namespaces.each do |namespace| + namespace.update_attribute(:traversal_ids, []) + end + end + + it 'reports the difference between the two tables' do + expected_result = { + next_start_id: Namespace.first.id + described_class::MAX_BATCHES * described_class::BATCH_SIZE, + batches: 4, + matches: 37, + mismatches: 3, + mismatches_details: different_namespaces.map do |namespace| + { + id: namespace.id, + source_table: [[]], + target_table: [[namespace.id]] # old traversal_ids of the namespace + } + end + } + expect(consistency_checker.execute(start_id: Namespace.first.id)).to eq(expected_result) + + expect(metrics_counter.get(source_table: "namespaces", result: "mismatch")).to eq(3) + expect(metrics_counter.get(source_table: "namespaces", result: "match")).to eq(37) + end + end + end +end diff --git a/spec/migrations/20220223124428_schedule_merge_topics_with_same_name_spec.rb b/spec/migrations/20220223124428_schedule_merge_topics_with_same_name_spec.rb new file mode 100644 index 00000000000..d9f6729475c --- /dev/null +++ b/spec/migrations/20220223124428_schedule_merge_topics_with_same_name_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe ScheduleMergeTopicsWithSameName do + let(:topics) { table(:topics) } + + describe '#up' do + before do + stub_const("#{described_class}::BATCH_SIZE", 2) + + topics.create!(name: 'topic1') + topics.create!(name: 'Topic2') + topics.create!(name: 'Topic3') + topics.create!(name: 'Topic4') + topics.create!(name: 'topic2') + topics.create!(name: 'topic3') + topics.create!(name: 'topic4') + topics.create!(name: 'TOPIC2') + topics.create!(name: 'topic5') + end + + it 'schedules MergeTopicsWithSameName background jobs', :aggregate_failures do + Sidekiq::Testing.fake! do + freeze_time do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, %w[topic2 topic3]) + expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, %w[topic4]) + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + end + end + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 8ed9c9051e1..1ad211b0c81 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6288,6 +6288,10 @@ RSpec.describe Project, factory_default: :keep do expect(subject.find_or_initialize_integration('prometheus')).to be_nil end + it 'returns nil if integration does not exist' do + expect(subject.find_or_initialize_integration('non-existing')).to be_nil + end + context 'with an existing integration' do subject { create(:project) } diff --git a/spec/services/database/consistency_check_service_spec.rb b/spec/services/database/consistency_check_service_spec.rb new file mode 100644 index 00000000000..2e642451432 --- /dev/null +++ b/spec/services/database/consistency_check_service_spec.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Database::ConsistencyCheckService do + let(:batch_size) { 5 } + let(:max_batches) { 2 } + + before do + stub_const("Gitlab::Database::ConsistencyChecker::BATCH_SIZE", batch_size) + stub_const("Gitlab::Database::ConsistencyChecker::MAX_BATCHES", max_batches) + end + + after do + redis_shared_state_cleanup! + end + + subject(:consistency_check_service) do + described_class.new( + source_model: Namespace, + target_model: Ci::NamespaceMirror, + source_columns: %w[id traversal_ids], + target_columns: %w[namespace_id traversal_ids] + ) + end + + describe '#random_start_id' do + let(:batch_size) { 5 } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + end + + it 'generates a random start_id within the records ids' do + 10.times do + start_id = subject.send(:random_start_id) + expect(start_id).to be_between(Namespace.first.id, Namespace.last.id).inclusive + end + end + end + + describe '#execute' do + let(:empty_results) do + { batches: 0, matches: 0, mismatches: 0, mismatches_details: [] } + end + + context 'when empty tables' do + it 'returns results with zero counters' do + result = consistency_check_service.execute + + expect(result).to eq(empty_results) + end + + it 'does not call the ConsistencyCheckService' do + expect(Gitlab::Database::ConsistencyChecker).not_to receive(:new) + consistency_check_service.execute + end + end + + context 'no cursor has been saved before' do + let(:selected_start_id) { Namespace.order(:id).limit(5).pluck(:id).last } + let(:expected_next_start_id) { selected_start_id + batch_size * max_batches } + + before do + create_list(:namespace, 50) # This will also create Ci::NameSpaceMirror objects + expect(consistency_check_service).to receive(:random_start_id).and_return(selected_start_id) + end + + it 'picks a random start_id' do + expected_result = { + batches: 2, + matches: 10, + mismatches: 0, + mismatches_details: [], + start_id: selected_start_id, + next_start_id: expected_next_start_id + } + expect(consistency_check_service.execute).to eq(expected_result) + end + + it 'calls the ConsistencyCheckService with the expected parameters' do + allow_next_instance_of(Gitlab::Database::ConsistencyChecker) do |instance| + expect(instance).to receive(:execute).with(start_id: selected_start_id).and_return({ + batches: 2, + next_start_id: expected_next_start_id, + matches: 10, + mismatches: 0, + mismatches_details: [] + }) + end + + expect(Gitlab::Database::ConsistencyChecker).to receive(:new).with( + source_model: Namespace, + target_model: Ci::NamespaceMirror, + source_columns: %w[id traversal_ids], + target_columns: %w[namespace_id traversal_ids] + ).and_call_original + + expected_result = { + batches: 2, + start_id: selected_start_id, + next_start_id: expected_next_start_id, + matches: 10, + mismatches: 0, + mismatches_details: [] + } + expect(consistency_check_service.execute).to eq(expected_result) + end + + it 'saves the next_start_id in Redis for he next iteration' do + expect(consistency_check_service).to receive(:save_next_start_id).with(expected_next_start_id).and_call_original + consistency_check_service.execute + end + end + + context 'cursor saved in Redis and moving' do + let(:first_namespace_id) { Namespace.order(:id).first.id } + let(:second_namespace_id) { Namespace.order(:id).second.id } + + before do + create_list(:namespace, 30) # This will also create Ci::NameSpaceMirror objects + end + + it "keeps moving the cursor with each call to the service" do + expect(consistency_check_service).to receive(:random_start_id).at_most(:once).and_return(first_namespace_id) + + allow_next_instance_of(Gitlab::Database::ConsistencyChecker) do |instance| + expect(instance).to receive(:execute).ordered.with(start_id: first_namespace_id).and_call_original + expect(instance).to receive(:execute).ordered.with(start_id: first_namespace_id + 10).and_call_original + expect(instance).to receive(:execute).ordered.with(start_id: first_namespace_id + 20).and_call_original + # Gets back to the start of the table + expect(instance).to receive(:execute).ordered.with(start_id: first_namespace_id).and_call_original + end + + 4.times do + consistency_check_service.execute + end + end + + it "keeps moving the cursor from any start point" do + expect(consistency_check_service).to receive(:random_start_id).at_most(:once).and_return(second_namespace_id) + + allow_next_instance_of(Gitlab::Database::ConsistencyChecker) do |instance| + expect(instance).to receive(:execute).ordered.with(start_id: second_namespace_id).and_call_original + expect(instance).to receive(:execute).ordered.with(start_id: second_namespace_id + 10).and_call_original + end + + 2.times do + consistency_check_service.execute + end + end + end + end +end diff --git a/spec/views/admin/application_settings/_repository_storage.html.haml_spec.rb b/spec/views/admin/application_settings/_repository_storage.html.haml_spec.rb index dc8f259eb56..244157a3b14 100644 --- a/spec/views/admin/application_settings/_repository_storage.html.haml_spec.rb +++ b/spec/views/admin/application_settings/_repository_storage.html.haml_spec.rb @@ -10,26 +10,7 @@ RSpec.describe 'admin/application_settings/_repository_storage.html.haml' do assign(:application_setting, app_settings) end - context 'additional storage config' do - let(:repository_storages_weighted) do - { - 'default' => 100, - 'mepmep' => 50 - } - end - - it 'lists them all' do - render - - Gitlab.config.repositories.storages.keys.each do |storage_name| - expect(rendered).to have_content(storage_name) - end - - expect(rendered).to have_content('foobar') - end - end - - context 'fewer storage configs' do + context 'with storage weights configured' do let(:repository_storages_weighted) do { 'default' => 100, @@ -38,14 +19,23 @@ RSpec.describe 'admin/application_settings/_repository_storage.html.haml' do } end + it 'lists storages with weight', :aggregate_failures do + render + + expect(rendered).to have_field('default', with: 100) + expect(rendered).to have_field('mepmep', with: 50) + end + + it 'lists storages without weight' do + render + + expect(rendered).to have_field('foobar', with: 0) + end + it 'lists only configured storages' do render - Gitlab.config.repositories.storages.keys.each do |storage_name| - expect(rendered).to have_content(storage_name) - end - - expect(rendered).not_to have_content('something_old') + expect(rendered).not_to have_field('something_old') end end end diff --git a/spec/workers/database/ci_namespace_mirrors_consistency_check_worker_spec.rb b/spec/workers/database/ci_namespace_mirrors_consistency_check_worker_spec.rb new file mode 100644 index 00000000000..116026ea8f7 --- /dev/null +++ b/spec/workers/database/ci_namespace_mirrors_consistency_check_worker_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Database::CiNamespaceMirrorsConsistencyCheckWorker do + let(:worker) { described_class.new } + + describe '#perform' do + context 'feature flag is disabled' do + before do + stub_feature_flags(ci_namespace_mirrors_consistency_check: false) + end + + it 'does not perform the consistency check on namespaces' do + expect(Database::ConsistencyCheckService).not_to receive(:new) + expect(worker).not_to receive(:log_extra_metadata_on_done) + worker.perform + end + end + + context 'feature flag is enabled' do + before do + stub_feature_flags(ci_namespace_mirrors_consistency_check: true) + end + + it 'executes the consistency check on namespaces' do + expect(Database::ConsistencyCheckService).to receive(:new).and_call_original + expected_result = { batches: 0, matches: 0, mismatches: 0, mismatches_details: [] } + expect(worker).to receive(:log_extra_metadata_on_done).with(:results, expected_result) + worker.perform + end + end + + context 'logs should contain the detailed mismatches' do + let(:first_namespace) { Namespace.all.order(:id).limit(1).first } + let(:missing_namespace) { Namespace.all.order(:id).limit(2).last } + + before do + redis_shared_state_cleanup! + stub_feature_flags(ci_namespace_mirrors_consistency_check: true) + create_list(:namespace, 10) # This will also create Ci::NameSpaceMirror objects + missing_namespace.delete + + allow_next_instance_of(Database::ConsistencyCheckService) do |instance| + allow(instance).to receive(:random_start_id).and_return(Namespace.first.id) + end + end + + it 'reports the differences to the logs' do + expected_result = { + batches: 1, + matches: 9, + mismatches: 1, + mismatches_details: [{ + id: missing_namespace.id, + source_table: nil, + target_table: [missing_namespace.traversal_ids] + }], + start_id: first_namespace.id, + next_start_id: first_namespace.id # The batch size > number of namespaces + } + expect(worker).to receive(:log_extra_metadata_on_done).with(:results, expected_result) + worker.perform + end + end + end +end diff --git a/spec/workers/database/ci_project_mirrors_consistency_check_worker_spec.rb b/spec/workers/database/ci_project_mirrors_consistency_check_worker_spec.rb new file mode 100644 index 00000000000..b6bd825ffcd --- /dev/null +++ b/spec/workers/database/ci_project_mirrors_consistency_check_worker_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Database::CiProjectMirrorsConsistencyCheckWorker do + let(:worker) { described_class.new } + + describe '#perform' do + context 'feature flag is disabled' do + before do + stub_feature_flags(ci_project_mirrors_consistency_check: false) + end + + it 'does not perform the consistency check on projects' do + expect(Database::ConsistencyCheckService).not_to receive(:new) + expect(worker).not_to receive(:log_extra_metadata_on_done) + worker.perform + end + end + + context 'feature flag is enabled' do + before do + stub_feature_flags(ci_project_mirrors_consistency_check: true) + end + + it 'executes the consistency check on projects' do + expect(Database::ConsistencyCheckService).to receive(:new).and_call_original + expected_result = { batches: 0, matches: 0, mismatches: 0, mismatches_details: [] } + expect(worker).to receive(:log_extra_metadata_on_done).with(:results, expected_result) + worker.perform + end + end + + context 'logs should contain the detailed mismatches' do + let(:first_project) { Project.all.order(:id).limit(1).first } + let(:missing_project) { Project.all.order(:id).limit(2).last } + + before do + redis_shared_state_cleanup! + stub_feature_flags(ci_project_mirrors_consistency_check: true) + create_list(:project, 10) # This will also create Ci::NameSpaceMirror objects + missing_project.delete + + allow_next_instance_of(Database::ConsistencyCheckService) do |instance| + allow(instance).to receive(:random_start_id).and_return(Project.first.id) + end + end + + it 'reports the differences to the logs' do + expected_result = { + batches: 1, + matches: 9, + mismatches: 1, + mismatches_details: [{ + id: missing_project.id, + source_table: nil, + target_table: [missing_project.namespace_id] + }], + start_id: first_project.id, + next_start_id: first_project.id # The batch size > number of projects + } + expect(worker).to receive(:log_extra_metadata_on_done).with(:results, expected_result) + worker.perform + end + end + end +end diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb index dcac1099687..0f007e970b4 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/product_intelligence.rb @@ -8,9 +8,9 @@ module Tooling REVIEW_LABEL = 'product intelligence::review pending' CHANGED_FILES_MESSAGE = <<~MSG For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/growth/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended - Please check the ~"product intelligence" [guide](https://docs.gitlab.com/ee/development/usage_ping.html). + Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). - For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/usage_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). + For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). %s