From 76f0d7fe6e1a80d5b07eaa792f33d62ec8736d0d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 16:43:19 +0900 Subject: [PATCH] Add background migration to fill file stores --- .../20180424151928_fill_file_store.rb.rb | 95 +++++++++++++++++++ .../fill_file_store_build_archive.rb | 21 ++++ .../fill_file_store_build_metadata.rb | 21 ++++ .../fill_file_store_job_artifact.rb | 20 ++++ .../fill_file_store_lfs_object.rb | 20 ++++ .../fill_file_store_upload.rb | 21 ++++ 6 files changed, 198 insertions(+) create mode 100644 db/post_migrate/20180424151928_fill_file_store.rb.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_build_archive.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_build_metadata.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_job_artifact.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_lfs_object.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_upload.rb diff --git a/db/post_migrate/20180424151928_fill_file_store.rb.rb b/db/post_migrate/20180424151928_fill_file_store.rb.rb new file mode 100644 index 00000000000..ecf169a4953 --- /dev/null +++ b/db/post_migrate/20180424151928_fill_file_store.rb.rb @@ -0,0 +1,95 @@ +class FillFileStore < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(artifacts_file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreBuildArchive', + 5.minutes, + batch_size: BATCH_SIZE) + end + + self.class.where(artifacts_metadata_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreBuildMetadata', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class JobArtifact < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_job_artifacts' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreJobArtifact', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class LfsObject < ActiveRecord::Base + include EachBatch + self.table_name = 'lfs_objects' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(file_store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreLfsObject', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + class Upload < ActiveRecord::Base + include EachBatch + self.table_name = 'uploads' + BATCH_SIZE = 10_000 + + def self.queue_background_migration + self.class.where(store: nil).tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + 'FillFileStoreUpload', + 5.minutes, + batch_size: BATCH_SIZE) + end + end + end + + def up + disable_statement_timeout + + # Schedule background migrations that fill 'NULL' value by '1' on `file_store`, `store`, `artifacts_file_store` columns + # '1' represents ObjectStorage::Store::LOCAL + # ci_builds.artifacts_file_store + # ci_builds.artifacts_metadata_store + # ci_job_artifacts.file_store + # lfs_objects.file_store + # uploads.store + + FillFileStore::Build.queue_background_migration + FillFileStore::JobArtifact.queue_background_migration + FillFileStore::LfsObject.queue_background_migration + FillFileStore::Upload.queue_background_migration + end + + def down + # noop + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_build_archive.rb b/lib/gitlab/background_migration/fill_file_store_build_archive.rb new file mode 100644 index 00000000000..02d744b7a74 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_build_archive.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreBuildArchive + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreBuildArchive::Build + .where('artifacts_file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(artifacts_file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_build_metadata.rb b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb new file mode 100644 index 00000000000..bf1a10b104e --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreBuildMetadata + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreBuildMetadata::Build + .where('artifacts_metadata_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(artifacts_metadata_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 00000000000..dfd1e7191d2 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where('file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 00000000000..d423bcfba95 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where('file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_upload.rb b/lib/gitlab/background_migration/fill_file_store_upload.rb new file mode 100644 index 00000000000..14f4c667ed8 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreUpload::Upload + .where('store = NULL') + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end