From 9c582b06476266e3e71dcabceb89ab0a47e18e5c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Sat, 29 Jan 2022 12:14:23 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- config/initializers/postgres_partitioning.rb | 3 +- ...ackground_migration_job_transition_logs.rb | 38 +++++++++++++++++++ db/schema_migrations/20211123135255 | 1 + db/structure.sql | 33 ++++++++++++++++ .../background_migration/batched_job.rb | 1 + .../batched_job_transition_log.rb | 25 ++++++++++++ lib/gitlab/database/gitlab_schemas.yml | 1 + .../background_migration/batched_job_spec.rb | 1 + .../batched_job_transition_log_spec.rb | 19 ++++++++++ 9 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20211123135255_create_batched_background_migration_job_transition_logs.rb create mode 100644 db/schema_migrations/20211123135255 create mode 100644 lib/gitlab/database/background_migration/batched_job_transition_log.rb create mode 100644 spec/lib/gitlab/database/background_migration/batched_job_transition_log_spec.rb diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index e44d5f445f6..074dbac64a5 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -3,7 +3,8 @@ Gitlab::Database::Partitioning.register_models([ AuditEvent, WebHookLog, - LooseForeignKeys::DeletedRecord + LooseForeignKeys::DeletedRecord, + Gitlab::Database::BackgroundMigration::BatchedJobTransitionLog ]) if Gitlab.ee? diff --git a/db/migrate/20211123135255_create_batched_background_migration_job_transition_logs.rb b/db/migrate/20211123135255_create_batched_background_migration_job_transition_logs.rb new file mode 100644 index 00000000000..280626c5d3b --- /dev/null +++ b/db/migrate/20211123135255_create_batched_background_migration_job_transition_logs.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class CreateBatchedBackgroundMigrationJobTransitionLogs < Gitlab::Database::Migration[1.0] + include Gitlab::Database::PartitioningMigrationHelpers + + def up + execute(<<~SQL) + CREATE TABLE batched_background_migration_job_transition_logs ( + id bigserial NOT NULL, + batched_background_migration_job_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + previous_status smallint NOT NULL, + next_status smallint NOT NULL, + exception_class text, + exception_message text, + CONSTRAINT check_50e580811a CHECK ((char_length(exception_message) <= 1000)), + CONSTRAINT check_76e202c37a CHECK ((char_length(exception_class) <= 100)), + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + + CREATE INDEX i_batched_background_migration_job_transition_logs_on_job_id + ON batched_background_migration_job_transition_logs USING btree (batched_background_migration_job_id); + + ALTER TABLE batched_background_migration_job_transition_logs ADD CONSTRAINT fk_rails_b7523a175b + FOREIGN KEY (batched_background_migration_job_id) REFERENCES batched_background_migration_jobs(id) ON DELETE CASCADE; + SQL + + min_date = Date.today + max_date = Date.today + 6.months + + create_daterange_partitions('batched_background_migration_job_transition_logs', 'created_at', min_date, max_date) + end + + def down + drop_table :batched_background_migration_job_transition_logs + end +end diff --git a/db/schema_migrations/20211123135255 b/db/schema_migrations/20211123135255 new file mode 100644 index 00000000000..e151abb0e33 --- /dev/null +++ b/db/schema_migrations/20211123135255 @@ -0,0 +1 @@ +cf6b9bb5711b6a097e399e79fdabe01a237581d99de7fed3c2b69c65ffd23a06 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index b4e534c3c41..6e0d49078e2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -127,6 +127,20 @@ CREATE TABLE audit_events ( ) PARTITION BY RANGE (created_at); +CREATE TABLE batched_background_migration_job_transition_logs ( + id bigint NOT NULL, + batched_background_migration_job_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + previous_status smallint NOT NULL, + next_status smallint NOT NULL, + exception_class text, + exception_message text, + CONSTRAINT check_50e580811a CHECK ((char_length(exception_message) <= 1000)), + CONSTRAINT check_76e202c37a CHECK ((char_length(exception_class) <= 100)) +) +PARTITION BY RANGE (created_at); + CREATE TABLE incident_management_pending_alert_escalations ( id bigint NOT NULL, rule_id bigint NOT NULL, @@ -10894,6 +10908,15 @@ CREATE TABLE banned_users ( user_id bigint NOT NULL ); +CREATE SEQUENCE batched_background_migration_job_transition_logs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE batched_background_migration_job_transition_logs_id_seq OWNED BY batched_background_migration_job_transition_logs.id; + CREATE TABLE batched_background_migration_jobs ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -21417,6 +21440,8 @@ ALTER TABLE ONLY background_migration_jobs ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY badges ALTER COLUMN id SET DEFAULT nextval('badges_id_seq'::regclass); +ALTER TABLE ONLY batched_background_migration_job_transition_logs ALTER COLUMN id SET DEFAULT nextval('batched_background_migration_job_transition_logs_id_seq'::regclass); + ALTER TABLE ONLY batched_background_migration_jobs ALTER COLUMN id SET DEFAULT nextval('batched_background_migration_jobs_id_seq'::regclass); ALTER TABLE ONLY batched_background_migrations ALTER COLUMN id SET DEFAULT nextval('batched_background_migrations_id_seq'::regclass); @@ -22833,6 +22858,9 @@ ALTER TABLE ONLY badges ALTER TABLE ONLY banned_users ADD CONSTRAINT banned_users_pkey PRIMARY KEY (user_id); +ALTER TABLE ONLY batched_background_migration_job_transition_logs + ADD CONSTRAINT batched_background_migration_job_transition_logs_pkey PRIMARY KEY (id, created_at); + ALTER TABLE ONLY batched_background_migration_jobs ADD CONSTRAINT batched_background_migration_jobs_pkey PRIMARY KEY (id); @@ -25075,6 +25103,8 @@ CREATE UNIQUE INDEX finding_link_url_idx ON vulnerability_finding_links USING bt CREATE INDEX finding_links_on_vulnerability_occurrence_id ON vulnerability_finding_links USING btree (vulnerability_occurrence_id); +CREATE INDEX i_batched_background_migration_job_transition_logs_on_job_id ON ONLY batched_background_migration_job_transition_logs USING btree (batched_background_migration_job_id); + CREATE UNIQUE INDEX i_ci_job_token_project_scope_links_on_source_and_target_project ON ci_job_token_project_scope_links USING btree (source_project_id, target_project_id); CREATE INDEX idx_analytics_devops_adoption_segments_on_namespace_id ON analytics_devops_adoption_segments USING btree (namespace_id); @@ -31083,6 +31113,9 @@ ALTER TABLE ONLY packages_debian_project_component_files ALTER TABLE ONLY namespace_aggregation_schedules ADD CONSTRAINT fk_rails_b565c8d16c FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; +ALTER TABLE batched_background_migration_job_transition_logs + ADD CONSTRAINT fk_rails_b7523a175b FOREIGN KEY (batched_background_migration_job_id) REFERENCES batched_background_migration_jobs(id) ON DELETE CASCADE; + ALTER TABLE ONLY approval_project_rules_protected_branches ADD CONSTRAINT fk_rails_b7567b031b FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE; diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 290fa51692a..e4db9391666 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -20,6 +20,7 @@ module Gitlab } belongs_to :batched_migration, foreign_key: :batched_background_migration_id + has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id scope :active, -> { where(status: [:pending, :running]) } scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } diff --git a/lib/gitlab/database/background_migration/batched_job_transition_log.rb b/lib/gitlab/database/background_migration/batched_job_transition_log.rb new file mode 100644 index 00000000000..1d034107041 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_job_transition_log.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedJobTransitionLog < ApplicationRecord + include PartitionedTable + + self.table_name = :batched_background_migration_job_transition_logs + + partitioned_by :created_at, strategy: :monthly, retain_for: 6.months + + belongs_to :batched_job, foreign_key: :batched_background_migration_job_id + + validates :previous_status, :next_status, :batched_job, presence: true + + validates :exception_class, length: { maximum: 100 } + validates :exception_message, length: { maximum: 1000 } + + enum previous_status: BatchedJob.statuses, _prefix: true + enum next_status: BatchedJob.statuses, _prefix: true + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index fb5d8cfa32f..90cd5d9d348 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -552,3 +552,4 @@ x509_commit_signatures: :gitlab_main x509_issuers: :gitlab_main zentao_tracker_data: :gitlab_main zoom_meetings: :gitlab_main +batched_background_migration_job_transition_logs: :gitlab_main diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb index c4364826ee2..d810cd2b570 100644 --- a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -7,6 +7,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d describe 'associations' do it { is_expected.to belong_to(:batched_migration).with_foreign_key(:batched_background_migration_id) } + it { is_expected.to have_many(:batched_job_transition_logs).with_foreign_key(:batched_background_migration_job_id) } end describe 'scopes' do diff --git a/spec/lib/gitlab/database/background_migration/batched_job_transition_log_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_transition_log_spec.rb new file mode 100644 index 00000000000..c42a0fc5e05 --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/batched_job_transition_log_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJobTransitionLog, type: :model do + describe 'associations' do + it { is_expected.to belong_to(:batched_job).with_foreign_key(:batched_background_migration_job_id) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:previous_status) } + it { is_expected.to validate_presence_of(:next_status) } + it { is_expected.to validate_presence_of(:batched_job) } + it { is_expected.to validate_length_of(:exception_class).is_at_most(100) } + it { is_expected.to validate_length_of(:exception_message).is_at_most(1000) } + it { is_expected.to define_enum_for(:previous_status).with_values(%i(pending running failed succeeded)).with_prefix } + it { is_expected.to define_enum_for(:next_status).with_values(%i(pending running failed succeeded)).with_prefix } + end +end