From 1eae7babcac271e94754f88c8f6df71e55e695de Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 21 Dec 2021 00:14:09 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/rails.gitlab-ci.yml | 6 + GITALY_SERVER_VERSION | 2 +- .../improved_container_scan_matching.yml | 8 - ...ra_finding_evidence_tables_foreign_keys.rb | 57 +++++ ...11_remove_extra_finding_evidence_tables.rb | 71 +++++++ db/schema_migrations/20211207173510 | 1 + db/schema_migrations/20211207173511 | 1 + db/structure.sql | 201 ------------------ .../background_migration/batched_job.rb | 18 +- .../background_migration/batched_migration.rb | 2 +- lib/gitlab/database/gitlab_schemas.yml | 6 - locale/gitlab.pot | 12 +- .../batched_migration_spec.rb | 27 ++- 13 files changed, 171 insertions(+), 241 deletions(-) delete mode 100644 config/feature_flags/development/improved_container_scan_matching.yml create mode 100644 db/post_migrate/20211207173510_remove_extra_finding_evidence_tables_foreign_keys.rb create mode 100644 db/post_migrate/20211207173511_remove_extra_finding_evidence_tables.rb create mode 100644 db/schema_migrations/20211207173510 create mode 100644 db/schema_migrations/20211207173511 diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index d676dc2f331..1d65437ba12 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -360,6 +360,12 @@ db:migrate:reset: script: - bundle exec rake db:migrate:reset +db:migrate:reset decomposed: + extends: + - db:migrate:reset + - .decomposed-database + - .rails:rules:decomposed-databases + db:migrate-from-previous-major-version: extends: .db-job-base variables: diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index b0c1791443b..4b482c71175 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -2db02e932c25bb96b269a9ac7a93f5bf52869865 +d19c6eab3ecc995ee5b947134657ebd92e35deb8 diff --git a/config/feature_flags/development/improved_container_scan_matching.yml b/config/feature_flags/development/improved_container_scan_matching.yml deleted file mode 100644 index 81a486831cc..00000000000 --- a/config/feature_flags/development/improved_container_scan_matching.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: improved_container_scan_matching -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73486 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344534 -milestone: '14.6' -type: development -group: group::container security -default_enabled: true diff --git a/db/post_migrate/20211207173510_remove_extra_finding_evidence_tables_foreign_keys.rb b/db/post_migrate/20211207173510_remove_extra_finding_evidence_tables_foreign_keys.rb new file mode 100644 index 00000000000..eaf07abe29d --- /dev/null +++ b/db/post_migrate/20211207173510_remove_extra_finding_evidence_tables_foreign_keys.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class RemoveExtraFindingEvidenceTablesForeignKeys < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + with_lock_retries do + remove_foreign_key :vulnerability_finding_evidence_assets, :vulnerability_finding_evidences + remove_foreign_key :vulnerability_finding_evidence_headers, :vulnerability_finding_evidence_requests + remove_foreign_key :vulnerability_finding_evidence_headers, :vulnerability_finding_evidence_responses + remove_foreign_key :vulnerability_finding_evidence_requests, :vulnerability_finding_evidences + remove_foreign_key :vulnerability_finding_evidence_requests, :vulnerability_finding_evidence_supporting_messages + remove_foreign_key :vulnerability_finding_evidence_responses, :vulnerability_finding_evidences + remove_foreign_key :vulnerability_finding_evidence_responses, :vulnerability_finding_evidence_supporting_messages + remove_foreign_key :vulnerability_finding_evidence_sources, :vulnerability_finding_evidences + remove_foreign_key :vulnerability_finding_evidence_supporting_messages, :vulnerability_finding_evidences + end + end + + def down + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_assets, :vulnerability_finding_evidences, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_headers, :vulnerability_finding_evidence_requests, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_headers, :vulnerability_finding_evidence_responses, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_requests, :vulnerability_finding_evidences, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_requests, :vulnerability_finding_evidence_supporting_messages, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_responses, :vulnerability_finding_evidences, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_responses, :vulnerability_finding_evidence_supporting_messages, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_sources, :vulnerability_finding_evidences, on_delete: :cascade + end + + with_lock_retries do + add_foreign_key :vulnerability_finding_evidence_supporting_messages, :vulnerability_finding_evidences, on_delete: :cascade + end + end +end diff --git a/db/post_migrate/20211207173511_remove_extra_finding_evidence_tables.rb b/db/post_migrate/20211207173511_remove_extra_finding_evidence_tables.rb new file mode 100644 index 00000000000..9a246a8c707 --- /dev/null +++ b/db/post_migrate/20211207173511_remove_extra_finding_evidence_tables.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +class RemoveExtraFindingEvidenceTables < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + with_lock_retries do + drop_table :vulnerability_finding_evidence_assets, if_exists: true + drop_table :vulnerability_finding_evidence_headers, if_exists: true + drop_table :vulnerability_finding_evidence_requests, if_exists: true + drop_table :vulnerability_finding_evidence_responses, if_exists: true + drop_table :vulnerability_finding_evidence_sources, if_exists: true + drop_table :vulnerability_finding_evidence_supporting_messages, if_exists: true + end + end + + def down + create_table :vulnerability_finding_evidence_assets, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence, index: { name: 'finding_evidence_assets_on_finding_evidence_id' }, null: false + t.text :type, limit: 2048 + t.text :name, limit: 2048 + t.text :url, limit: 2048 + end + + create_table :vulnerability_finding_evidence_sources, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence, index: { name: 'finding_evidence_sources_on_finding_evidence_id' }, null: false + t.text :name, limit: 2048 + t.text :url, limit: 2048 + end + + create_table :vulnerability_finding_evidence_supporting_messages, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence, index: { name: 'finding_evidence_supporting_messages_on_finding_evidence_id' }, null: false + t.text :name, limit: 2048 + end + + create_table :vulnerability_finding_evidence_requests, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence, index: { name: 'finding_evidence_requests_on_finding_evidence_id' }, null: true + t.text :method, limit: 32 + t.text :url, limit: 2048 + t.text :body, limit: 2048 + t.references :vulnerability_finding_evidence_supporting_message, index: { name: 'finding_evidence_requests_on_supporting_evidence_id' }, null: true + end + + create_table :vulnerability_finding_evidence_responses, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence, index: { name: 'finding_evidence_responses_on_finding_evidences_id' }, null: true + t.integer :status_code + t.text :reason_phrase, limit: 2048 + t.text :body, limit: 2048 + t.references :vulnerability_finding_evidence_supporting_message, index: { name: 'finding_evidence_responses_on_supporting_evidence_id' }, null: true + end + + create_table :vulnerability_finding_evidence_headers, if_not_exists: true do |t| + t.timestamps_with_timezone null: false + + t.references :vulnerability_finding_evidence_request, index: { name: 'finding_evidence_header_on_finding_evidence_request_id' }, null: true + t.references :vulnerability_finding_evidence_response, index: { name: 'finding_evidence_header_on_finding_evidence_response_id' }, null: true + t.text :name, null: false, limit: 255 + t.text :value, null: false, limit: 8192 + end + end +end diff --git a/db/schema_migrations/20211207173510 b/db/schema_migrations/20211207173510 new file mode 100644 index 00000000000..09474e1c643 --- /dev/null +++ b/db/schema_migrations/20211207173510 @@ -0,0 +1 @@ +0a4ac9de84b8351f39e549904d9e661648b496e6e3183c4ff5eb22b70d5ba7e9 \ No newline at end of file diff --git a/db/schema_migrations/20211207173511 b/db/schema_migrations/20211207173511 new file mode 100644 index 00000000000..93b44c20390 --- /dev/null +++ b/db/schema_migrations/20211207173511 @@ -0,0 +1 @@ +8f41f45c5ef23eafae2e67951497b5752f4b30ecf73ae3c08f61febfa4fb17be \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 24807a9e475..1a8c1d2aeff 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -20631,132 +20631,6 @@ CREATE SEQUENCE vulnerability_feedback_id_seq ALTER SEQUENCE vulnerability_feedback_id_seq OWNED BY vulnerability_feedback.id; -CREATE TABLE vulnerability_finding_evidence_assets ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_id bigint NOT NULL, - type text, - name text, - url text, - CONSTRAINT check_5adf5d69de CHECK ((char_length(type) <= 2048)), - CONSTRAINT check_839f29d7ca CHECK ((char_length(name) <= 2048)), - CONSTRAINT check_9272d912c0 CHECK ((char_length(url) <= 2048)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_assets_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_assets_id_seq OWNED BY vulnerability_finding_evidence_assets.id; - -CREATE TABLE vulnerability_finding_evidence_headers ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_request_id bigint, - vulnerability_finding_evidence_response_id bigint, - name text NOT NULL, - value text NOT NULL, - CONSTRAINT check_01d21e8d92 CHECK ((char_length(name) <= 255)), - CONSTRAINT check_3f9011f903 CHECK ((char_length(value) <= 8192)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_headers_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_headers_id_seq OWNED BY vulnerability_finding_evidence_headers.id; - -CREATE TABLE vulnerability_finding_evidence_requests ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_id bigint, - method text, - url text, - body text, - vulnerability_finding_evidence_supporting_message_id bigint, - CONSTRAINT check_7e37f2d01a CHECK ((char_length(body) <= 2048)), - CONSTRAINT check_8152fbb236 CHECK ((char_length(url) <= 2048)), - CONSTRAINT check_d9d11300f4 CHECK ((char_length(method) <= 32)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_requests_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_requests_id_seq OWNED BY vulnerability_finding_evidence_requests.id; - -CREATE TABLE vulnerability_finding_evidence_responses ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_id bigint, - status_code integer, - reason_phrase text, - body text, - vulnerability_finding_evidence_supporting_message_id bigint, - CONSTRAINT check_58b124ab48 CHECK ((char_length(reason_phrase) <= 2048)), - CONSTRAINT check_76bac0c32b CHECK ((char_length(body) <= 2048)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_responses_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_responses_id_seq OWNED BY vulnerability_finding_evidence_responses.id; - -CREATE TABLE vulnerability_finding_evidence_sources ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_id bigint NOT NULL, - name text, - url text, - CONSTRAINT check_0fe01298d6 CHECK ((char_length(url) <= 2048)), - CONSTRAINT check_86b537ba1a CHECK ((char_length(name) <= 2048)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_sources_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_sources_id_seq OWNED BY vulnerability_finding_evidence_sources.id; - -CREATE TABLE vulnerability_finding_evidence_supporting_messages ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - vulnerability_finding_evidence_id bigint NOT NULL, - name text, - CONSTRAINT check_fa33b9ae85 CHECK ((char_length(name) <= 2048)) -); - -CREATE SEQUENCE vulnerability_finding_evidence_supporting_messages_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_finding_evidence_supporting_messages_id_seq OWNED BY vulnerability_finding_evidence_supporting_messages.id; - CREATE TABLE vulnerability_finding_evidences ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -22244,18 +22118,6 @@ ALTER TABLE ONLY vulnerability_external_issue_links ALTER COLUMN id SET DEFAULT ALTER TABLE ONLY vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('vulnerability_feedback_id_seq'::regclass); -ALTER TABLE ONLY vulnerability_finding_evidence_assets ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_assets_id_seq'::regclass); - -ALTER TABLE ONLY vulnerability_finding_evidence_headers ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_headers_id_seq'::regclass); - -ALTER TABLE ONLY vulnerability_finding_evidence_requests ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_requests_id_seq'::regclass); - -ALTER TABLE ONLY vulnerability_finding_evidence_responses ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_responses_id_seq'::regclass); - -ALTER TABLE ONLY vulnerability_finding_evidence_sources ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_sources_id_seq'::regclass); - -ALTER TABLE ONLY vulnerability_finding_evidence_supporting_messages ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidence_supporting_messages_id_seq'::regclass); - ALTER TABLE ONLY vulnerability_finding_evidences ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidences_id_seq'::regclass); ALTER TABLE ONLY vulnerability_finding_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_links_id_seq'::regclass); @@ -24275,24 +24137,6 @@ ALTER TABLE ONLY vulnerability_external_issue_links ALTER TABLE ONLY vulnerability_feedback ADD CONSTRAINT vulnerability_feedback_pkey PRIMARY KEY (id); -ALTER TABLE ONLY vulnerability_finding_evidence_assets - ADD CONSTRAINT vulnerability_finding_evidence_assets_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY vulnerability_finding_evidence_headers - ADD CONSTRAINT vulnerability_finding_evidence_headers_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY vulnerability_finding_evidence_requests - ADD CONSTRAINT vulnerability_finding_evidence_requests_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY vulnerability_finding_evidence_responses - ADD CONSTRAINT vulnerability_finding_evidence_responses_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY vulnerability_finding_evidence_sources - ADD CONSTRAINT vulnerability_finding_evidence_sources_pkey PRIMARY KEY (id); - -ALTER TABLE ONLY vulnerability_finding_evidence_supporting_messages - ADD CONSTRAINT vulnerability_finding_evidence_supporting_messages_pkey PRIMARY KEY (id); - ALTER TABLE ONLY vulnerability_finding_evidences ADD CONSTRAINT vulnerability_finding_evidences_pkey PRIMARY KEY (id); @@ -25072,24 +24916,6 @@ CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_and_note_id_index ON epic_user CREATE UNIQUE INDEX epic_user_mentions_on_epic_id_index ON epic_user_mentions USING btree (epic_id) WHERE (note_id IS NULL); -CREATE INDEX finding_evidence_assets_on_finding_evidence_id ON vulnerability_finding_evidence_assets USING btree (vulnerability_finding_evidence_id); - -CREATE INDEX finding_evidence_header_on_finding_evidence_request_id ON vulnerability_finding_evidence_headers USING btree (vulnerability_finding_evidence_request_id); - -CREATE INDEX finding_evidence_header_on_finding_evidence_response_id ON vulnerability_finding_evidence_headers USING btree (vulnerability_finding_evidence_response_id); - -CREATE INDEX finding_evidence_requests_on_finding_evidence_id ON vulnerability_finding_evidence_requests USING btree (vulnerability_finding_evidence_id); - -CREATE INDEX finding_evidence_requests_on_supporting_evidence_id ON vulnerability_finding_evidence_requests USING btree (vulnerability_finding_evidence_supporting_message_id); - -CREATE INDEX finding_evidence_responses_on_finding_evidences_id ON vulnerability_finding_evidence_responses USING btree (vulnerability_finding_evidence_id); - -CREATE INDEX finding_evidence_responses_on_supporting_evidence_id ON vulnerability_finding_evidence_responses USING btree (vulnerability_finding_evidence_supporting_message_id); - -CREATE INDEX finding_evidence_sources_on_finding_evidence_id ON vulnerability_finding_evidence_sources USING btree (vulnerability_finding_evidence_id); - -CREATE INDEX finding_evidence_supporting_messages_on_finding_evidence_id ON vulnerability_finding_evidence_supporting_messages USING btree (vulnerability_finding_evidence_id); - CREATE UNIQUE INDEX finding_evidences_on_unique_vulnerability_occurrence_id ON vulnerability_finding_evidences USING btree (vulnerability_occurrence_id); CREATE INDEX finding_links_on_vulnerability_occurrence_id ON vulnerability_finding_links USING btree (vulnerability_occurrence_id); @@ -30269,9 +30095,6 @@ ALTER TABLE ONLY service_desk_settings ALTER TABLE ONLY saml_group_links ADD CONSTRAINT fk_rails_22e312c530 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_responses - ADD CONSTRAINT fk_rails_2390a09723 FOREIGN KEY (vulnerability_finding_evidence_id) REFERENCES vulnerability_finding_evidences(id) ON DELETE CASCADE; - ALTER TABLE ONLY dast_profiles ADD CONSTRAINT fk_rails_23cae5abe1 FOREIGN KEY (dast_scanner_profile_id) REFERENCES dast_scanner_profiles(id) ON DELETE CASCADE; @@ -30728,9 +30551,6 @@ ALTER TABLE ONLY vulnerability_findings_remediations ALTER TABLE ONLY resource_iteration_events ADD CONSTRAINT fk_rails_6830c13ac1 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_headers - ADD CONSTRAINT fk_rails_683b8e000c FOREIGN KEY (vulnerability_finding_evidence_response_id) REFERENCES vulnerability_finding_evidence_responses(id) ON DELETE CASCADE; - ALTER TABLE ONLY geo_hashed_storage_migrated_events ADD CONSTRAINT fk_rails_687ed7d7c5 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -30752,9 +30572,6 @@ ALTER TABLE ONLY prometheus_alerts ALTER TABLE ONLY term_agreements ADD CONSTRAINT fk_rails_6ea6520e4a FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_assets - ADD CONSTRAINT fk_rails_6edbbecba4 FOREIGN KEY (vulnerability_finding_evidence_id) REFERENCES vulnerability_finding_evidences(id) ON DELETE CASCADE; - ALTER TABLE ONLY project_compliance_framework_settings ADD CONSTRAINT fk_rails_6f5294f16c FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -30785,9 +30602,6 @@ ALTER TABLE ONLY dast_scanner_profiles ALTER TABLE ONLY vulnerability_historical_statistics ADD CONSTRAINT fk_rails_72b73ed023 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_requests - ADD CONSTRAINT fk_rails_72c87c8eb6 FOREIGN KEY (vulnerability_finding_evidence_id) REFERENCES vulnerability_finding_evidences(id) ON DELETE CASCADE; - ALTER TABLE ONLY slack_integrations ADD CONSTRAINT fk_rails_73db19721a FOREIGN KEY (service_id) REFERENCES integrations(id) ON DELETE CASCADE; @@ -30842,9 +30656,6 @@ ALTER TABLE ONLY terraform_states ALTER TABLE ONLY analytics_cycle_analytics_project_stages ADD CONSTRAINT fk_rails_796a7dbc9c FOREIGN KEY (project_value_stream_id) REFERENCES analytics_cycle_analytics_project_value_streams(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_supporting_messages - ADD CONSTRAINT fk_rails_79e77f6c5c FOREIGN KEY (vulnerability_finding_evidence_id) REFERENCES vulnerability_finding_evidences(id) ON DELETE CASCADE; - ALTER TABLE ONLY software_license_policies ADD CONSTRAINT fk_rails_7a7a2a92de FOREIGN KEY (software_license_id) REFERENCES software_licenses(id) ON DELETE CASCADE; @@ -30971,9 +30782,6 @@ ALTER TABLE ONLY project_error_tracking_settings ALTER TABLE ONLY list_user_preferences ADD CONSTRAINT fk_rails_916d72cafd FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_responses - ADD CONSTRAINT fk_rails_929041a499 FOREIGN KEY (vulnerability_finding_evidence_supporting_message_id) REFERENCES vulnerability_finding_evidence_supporting_messages(id) ON DELETE CASCADE; - ALTER TABLE ONLY merge_request_cleanup_schedules ADD CONSTRAINT fk_rails_92dd0e705c FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; @@ -31334,18 +31142,12 @@ ALTER TABLE ONLY operations_strategies_user_lists ALTER TABLE ONLY issue_tracker_data ADD CONSTRAINT fk_rails_ccc0840427 FOREIGN KEY (service_id) REFERENCES integrations(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_headers - ADD CONSTRAINT fk_rails_ce7f121a03 FOREIGN KEY (vulnerability_finding_evidence_request_id) REFERENCES vulnerability_finding_evidence_requests(id) ON DELETE CASCADE; - ALTER TABLE ONLY resource_milestone_events ADD CONSTRAINT fk_rails_cedf8cce4d FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL; ALTER TABLE ONLY resource_iteration_events ADD CONSTRAINT fk_rails_cee126f66c FOREIGN KEY (iteration_id) REFERENCES sprints(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_requests - ADD CONSTRAINT fk_rails_cf0f278cb0 FOREIGN KEY (vulnerability_finding_evidence_supporting_message_id) REFERENCES vulnerability_finding_evidence_supporting_messages(id) ON DELETE CASCADE; - ALTER TABLE ONLY upload_states ADD CONSTRAINT fk_rails_d00f153613 FOREIGN KEY (upload_id) REFERENCES uploads(id) ON DELETE CASCADE; @@ -31502,9 +31304,6 @@ ALTER TABLE ONLY vulnerability_issue_links ALTER TABLE ONLY merge_request_blocks ADD CONSTRAINT fk_rails_e9387863bc FOREIGN KEY (blocking_merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_finding_evidence_sources - ADD CONSTRAINT fk_rails_e9761bed4c FOREIGN KEY (vulnerability_finding_evidence_id) REFERENCES vulnerability_finding_evidences(id) ON DELETE CASCADE; - ALTER TABLE ONLY protected_branch_unprotect_access_levels ADD CONSTRAINT fk_rails_e9eb8dc025 FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE; diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 503172dd750..290fa51692a 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -12,17 +12,6 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze - belongs_to :batched_migration, foreign_key: :batched_background_migration_id - - scope :active, -> { where(status: [:pending, :running]) } - scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } - scope :retriable, -> { - failed_jobs = where(status: :failed).where('attempts < ?', MAX_ATTEMPTS) - - from_union([failed_jobs, self.stuck]) - } - scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } - enum status: { pending: 0, running: 1, @@ -30,7 +19,14 @@ module Gitlab succeeded: 3 } + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + + scope :active, -> { where(status: [:pending, :running]) } + scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } + scope :retriable, -> { from_union([failed.where('attempts < ?', MAX_ATTEMPTS), self.stuck]) } + scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + scope :with_preloads, -> { preload(:batched_migration) } delegate :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 2844cbe4a74..2f066039874 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -113,7 +113,7 @@ module Gitlab end def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) - jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs) + jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs).with_preloads return if jobs.size < number_of_jobs diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 24c2d634780..c9096de6b6e 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -521,13 +521,7 @@ vulnerabilities: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main -vulnerability_finding_evidence_assets: :gitlab_main -vulnerability_finding_evidence_headers: :gitlab_main -vulnerability_finding_evidence_requests: :gitlab_main -vulnerability_finding_evidence_responses: :gitlab_main vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_evidence_sources: :gitlab_main -vulnerability_finding_evidence_supporting_messages: :gitlab_main vulnerability_finding_links: :gitlab_main vulnerability_finding_signatures: :gitlab_main vulnerability_findings_remediations: :gitlab_main diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 076b54a12e6..ed0371f3f89 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -19577,9 +19577,6 @@ msgstr "" msgid "Is blocked by" msgstr "" -msgid "Is this GitLab trial for your company?" -msgstr "" - msgid "Is using license seat:" msgstr "" @@ -26464,6 +26461,9 @@ msgstr "" msgid "Please select a file" msgstr "" +msgid "Please select a group" +msgstr "" + msgid "Please select a group." msgstr "" @@ -37289,15 +37289,15 @@ msgstr "" msgid "Trial|How many employees will use Gitlab?" msgstr "" -msgid "Trial|How many users will be evaluating the trial?" -msgstr "" - msgid "Trial|Last name" msgstr "" msgid "Trial|Number of employees" msgstr "" +msgid "Trial|Number of trial users" +msgstr "" + msgid "Trial|Please select a country" msgstr "" diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb index 49714cfc4dd..01d61a525e6 100644 --- a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -336,8 +336,8 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end describe '#smoothed_time_efficiency' do - let(:migration) { create(:batched_background_migration, interval: 120.seconds) } - let(:end_time) { Time.zone.now } + let_it_be(:migration) { create(:batched_background_migration, interval: 120.seconds) } + let_it_be(:end_time) { Time.zone.now } around do |example| freeze_time do @@ -345,7 +345,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end - let(:common_attrs) do + let_it_be(:common_attrs) do { status: :succeeded, batched_migration: migration, @@ -364,13 +364,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end context 'when there are enough jobs' do + let_it_be(:number_of_jobs) { 10 } + let_it_be(:jobs) { create_list(:batched_background_migration_job, number_of_jobs, **common_attrs.merge(batched_migration: migration)) } + subject { migration.smoothed_time_efficiency(number_of_jobs: number_of_jobs) } - let!(:jobs) { create_list(:batched_background_migration_job, number_of_jobs, **common_attrs.merge(batched_migration: migration)) } - let(:number_of_jobs) { 10 } - before do - expect(migration).to receive_message_chain(:batched_jobs, :successful_in_execution_order, :reverse_order, :limit).with(no_args).with(no_args).with(number_of_jobs).and_return(jobs) + expect(migration).to receive_message_chain(:batched_jobs, :successful_in_execution_order, :reverse_order, :limit, :with_preloads) + .and_return(jobs) end def mock_efficiencies(*effs) @@ -411,6 +412,18 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m end end end + + context 'with preloaded batched migration' do + it 'avoids N+1' do + create_list(:batched_background_migration_job, 11, **common_attrs.merge(started_at: end_time - 10.seconds)) + + control = ActiveRecord::QueryRecorder.new do + migration.smoothed_time_efficiency(number_of_jobs: 10) + end + + expect { migration.smoothed_time_efficiency(number_of_jobs: 11) }.not_to exceed_query_limit(control) + end + end end describe '#optimize!' do