gitlab-org--gitlab-foss/spec/migrations
Heinrich Lee Yu a8a4518099 Clean up noteable_id for notes on commits
This was incorrectly set by a bug in:
https://gitlab.com/gitlab-org/gitlab-ce/issues/54924

Also adds a `batch_size` option to `update_column_in_batches`
2019-04-06 08:29:51 +08:00
..
active_record
add_foreign_key_from_notification_settings_to_users_spec.rb
add_foreign_key_to_merge_requests_spec.rb
add_foreign_keys_to_todos_spec.rb Remove factories from AddForeignKeysToTodos spec 2019-03-12 09:58:26 +00:00
add_head_pipeline_for_each_merge_request_spec.rb Remove factories from AddHeadPipelineForEachMergeRequest spec 2019-03-14 11:34:49 +00:00
add_not_null_constraint_to_project_mirror_data_foreign_key_spec.rb
add_pages_access_level_to_project_feature_spec.rb Make GitLab pages support access control 2018-10-05 13:41:11 +00:00
add_pipeline_build_foreign_key_spec.rb
add_unique_constraint_to_project_features_project_id_spec.rb
assure_commits_count_for_merge_request_diff_spec.rb
backfill_releases_name_with_tag_name_spec.rb Add name, author and sha to releases 2018-12-13 12:15:21 +01:00
backfill_store_project_full_path_in_repo_spec.rb Always run CleanUp before writing the git config 2018-11-27 22:48:55 +01:00
calculate_conv_dev_index_percentages_spec.rb Remove factories from CalculateConvDevIndexPercentages spec 2019-03-12 10:07:44 +00:00
change_default_value_for_dsa_key_restriction_spec.rb
clean_appearance_symlinks_spec.rb
clean_stage_id_reference_migration_spec.rb
clean_stages_statuses_migration_spec.rb
clean_up_for_members_spec.rb Fix migration failure 2019-02-07 15:39:29 +01:00
clean_up_noteable_id_for_notes_on_commits_spec.rb Clean up noteable_id for notes on commits 2019-04-06 08:29:51 +08:00
clean_upload_symlinks_spec.rb
cleanup_build_stage_migration_spec.rb
cleanup_environments_external_url_spec.rb Merge branch 'security-stored-xss-for-environments' into 'master' 2018-11-28 19:07:29 -05:00
cleanup_legacy_artifact_migration_spec.rb Cleanup legacy artifact background migration 2019-01-17 14:01:50 +09:00
cleanup_move_system_upload_folder_symlink_spec.rb
cleanup_namespaceless_pending_delete_projects_spec.rb Fix RSpec/FactoriesInMigrationSpecs in two files 2019-01-15 19:08:06 +01:00
cleanup_nonexisting_namespace_pending_delete_projects_spec.rb Remove factories from projects cleanup spec 2019-03-12 14:09:30 +00:00
cleanup_stages_position_migration_spec.rb
convert_custom_notification_settings_to_columns_spec.rb
create_missing_namespace_for_internal_users_spec.rb
delete_conflicting_redirect_routes_spec.rb
delete_inconsistent_internal_id_records_spec.rb Reduce differences with EE in a migration spec 2019-03-14 16:55:52 +01:00
drop_duplicate_protected_tags_spec.rb
enqueue_delete_diff_files_workers_spec.rb
enqueue_redact_links_spec.rb Redact unsubscribe links in issuable texts 2018-10-23 21:20:20 +02:00
enqueue_verify_pages_domain_workers_spec.rb
fill_empty_finished_at_in_deployments_spec.rb Squashed commit of the following: 2018-11-06 10:54:27 +09:00
fill_file_store_spec.rb
fix_null_type_labels_spec.rb Fix template labels 2019-02-04 10:52:19 +00:00
fix_wrongly_renamed_routes_spec.rb
generate_missing_routes_spec.rb
import_common_metrics_spec.rb
issues_moved_to_id_foreign_key_spec.rb Remove factories from IssuesMovedToIdForeignKey spec 2019-03-12 14:09:30 +00:00
migrate_auto_dev_ops_domain_to_cluster_domain_spec.rb Autocorrect with RSpec/ExampleWording cop 2019-04-05 08:43:27 +00:00
migrate_build_stage_reference_again_spec.rb
migrate_cluster_configure_worker_sidekiq_queue_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_create_trace_artifact_sidekiq_queue_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_forbidden_redirect_uris_spec.rb Merge branch 'security-fix-uri-xss-applications' into 'master' 2018-11-28 19:14:15 -05:00
migrate_gcp_clusters_to_new_clusters_architectures_spec.rb
migrate_import_attributes_data_from_projects_to_project_mirror_data_spec.rb
migrate_issues_to_ghost_user_spec.rb Make it only run the associated migration 2018-11-27 22:48:55 +01:00
migrate_kubernetes_service_to_new_clusters_architectures_spec.rb
migrate_legacy_artifacts_to_job_artifacts_spec.rb
migrate_null_wiki_access_levels_spec.rb
migrate_object_storage_upload_sidekiq_queue_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_old_artifacts_spec.rb Remove factories from MigrateOldArtifacts spec 2019-03-14 15:51:56 +00:00
migrate_pipeline_sidekiq_queues_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_pipeline_stages_spec.rb
migrate_process_commit_worker_jobs_spec.rb
migrate_remaining_mr_metrics_populating_background_migration_spec.rb
migrate_stage_id_reference_in_background_spec.rb
migrate_stages_statuses_spec.rb
migrate_storage_migrator_sidekiq_queue_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_update_head_pipeline_for_merge_request_sidekiq_queue_spec.rb Refactor subbed_worker calls 2019-03-28 15:50:47 +00:00
migrate_user_activities_to_users_last_activity_on_spec.rb Remove factories from user activities migration spec 2019-03-14 15:51:56 +00:00
migrate_user_authentication_token_to_personal_access_token_spec.rb
migrate_user_project_view_spec.rb Remove factories from MigrateUserProjectView spec 2019-03-14 15:51:56 +00:00
move_personal_snippets_files_spec.rb Remove factories from MovePersonalSnippetsFiles spec 2019-03-12 09:56:07 +00:00
move_system_upload_folder_spec.rb
move_uploads_to_system_dir_spec.rb
normalize_ldap_extern_uids_spec.rb
populate_can_push_from_deploy_keys_projects_spec.rb
populate_mr_metrics_with_events_data_spec.rb Populate MR metrics with events table information (migration) 2018-12-06 22:08:38 -02:00
README.md Reorders migrations readme file 2019-01-17 18:11:10 -06:00
remove_assignee_id_from_issue_spec.rb
remove_dot_git_from_usernames_spec.rb
remove_duplicate_mr_events_spec.rb
remove_empty_extern_uid_auth0_identities_spec.rb
remove_empty_fork_networks_spec.rb
remove_orphaned_label_links_spec.rb Remove orphaned label links 2018-09-07 07:38:44 -07:00
remove_project_labels_group_id_spec.rb
remove_redundant_pipeline_stages_spec.rb
remove_soft_removed_objects_spec.rb
rename_duplicated_variable_key_spec.rb
rename_more_reserved_project_names_spec.rb Refactored AfterRenameService to reduce coupling 2019-01-22 13:55:18 +01:00
rename_reserved_project_names_spec.rb Refactored AfterRenameService to reduce coupling 2019-01-22 13:55:18 +01:00
rename_users_with_renamed_namespace_spec.rb
reschedule_builds_stages_migration_spec.rb
reschedule_commits_count_for_merge_request_diff_spec.rb
schedule_create_gpg_key_subkeys_from_gpg_keys_spec.rb
schedule_digest_personal_access_tokens_spec.rb [master] Persist only SHA digest of PersonalAccessToken#token 2018-10-29 16:06:45 +00:00
schedule_merge_request_diff_migrations_spec.rb
schedule_merge_request_diff_migrations_take_two_spec.rb
schedule_merge_request_latest_merge_request_diff_id_migrations_spec.rb
schedule_populate_merge_request_assignees_table_spec.rb Add multiple assignees migration and table population 2019-03-25 19:50:40 -03:00
schedule_populate_merge_request_metrics_with_events_data_spec.rb
schedule_runners_token_encryption_spec.rb Fix an order-dependent spec failure in spec/migrations/schedule_sync_issuables_state_id_spec.rb 2019-04-04 14:12:10 +01:00
schedule_set_confidential_note_events_on_webhooks_spec.rb
schedule_stages_index_migration_spec.rb
schedule_sync_issuables_state_id_spec.rb Fix an order-dependent spec failure in spec/migrations/schedule_sync_issuables_state_id_spec.rb 2019-04-04 14:12:10 +01:00
schedule_to_archive_legacy_traces_spec.rb
steal_fill_store_upload_spec.rb Add migration to steal FillStoreUpload 2018-11-07 11:29:31 +01:00
track_untracked_uploads_spec.rb
truncate_user_fullname_spec.rb Add limit of 128 characters to users name 2019-04-05 09:35:34 +02:00
turn_nested_groups_into_regular_groups_for_mysql_spec.rb
update_legacy_diff_notes_type_for_import_spec.rb
update_notes_type_for_import_spec.rb
update_project_import_visibility_level_spec.rb Fix tree restorer visibility level 2019-01-31 16:52:49 +01:00
update_retried_for_ci_build_spec.rb
update_upload_paths_to_system_spec.rb

Testing migrations

In order to reliably test a migration, we need to test it against a database schema that this migration has been written for. In order to achieve that we have some migration helpers and RSpec test tag, called :migration.

If you want to write a test for a migration consider adding :migration tag to the test signature, like describe SomeMigrationClass, :migration.

How does it work?

Adding a :migration tag to a test signature injects a few before / after hooks to the test.

The most important change is that adding a :migration tag adds a before hook that will revert all migrations to the point that a migration under test is not yet migrated.

In other words, our custom RSpec hooks will find a previous migration, and migrate the database down to the previous migration version.

With this approach you can test a migration against a database schema that this migration has been written for.

The after hook will migrate the database up and reinstitutes the latest schema version, so that the process does not affect subsequent specs and ensures proper isolation.

Available helpers

Use table helper to create a temporary ActiveRecord::Base derived model for a table.

See spec/support/helpers/migrations_helpers.rb for all the available helpers.

Testing a class that is an ActiveRecord::Migration

In order to test a class that is an ActiveRecord::Migration, you will need to manually require the migration file because it is not autoloaded with Rails.

Use migrate! helper to run the migration that is under test. It will not only run migration, but will also bump the schema version in the schema_migrations table. It is necessary because in the after hook we trigger the rest of the migrations, and we need to know where to start.

Example

This spec tests the db/post_migrate/20170526185842_migrate_pipeline_stages.rb migration. You can find the complete spec on spec/migrations/migrate_pipeline_stages_spec.rb.

require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb')

describe MigratePipelineStages, :migration do

  # Create test data - pipeline and CI/CD jobs.

  let(:jobs) { table(:ci_builds) }
  let(:stages) { table(:ci_stages) }
  let(:pipelines) { table(:ci_pipelines) }
  let(:projects) { table(:projects) }

  before do
    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
  end

  # Test the migration.

  it 'correctly migrates pipeline stages' do
    expect(stages.count).to be_zero

    migrate!

    expect(stages.count).to eq 2
    expect(stages.all.pluck(:name)).to match_array %w[test build]
  end
end

Testing a class that is not an ActiveRecord::Migration

To test a class that is not an ActiveRecord::Migration (a background migration), you will need to manually provide a required schema version. Please add a schema tag to a context that you want to switch the database schema within.

Example: describe SomeClass, :migration, schema: 20170608152748.

Example

This spec tests the lib/gitlab/background_migration/archive_legacy_traces.rb background migration. You can find the complete spec on spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb

require 'spec_helper'

describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do
  include TraceHelpers

  let(:namespaces) { table(:namespaces) }
  let(:projects) { table(:projects) }
  let(:builds) { table(:ci_builds) }
  let(:job_artifacts) { table(:ci_job_artifacts) }

  before do
    namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
    @build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build')
  end

  context 'when trace file exsits at the right place' do
    before do
      create_legacy_trace(@build, 'trace in file')
    end

    it 'correctly archive legacy traces' do
      expect(job_artifacts.count).to eq(0)
      expect(File.exist?(legacy_trace_path(@build))).to be_truthy

      described_class.new.perform(1, 1)

      expect(job_artifacts.count).to eq(1)
      expect(File.exist?(legacy_trace_path(@build))).to be_falsy
      expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file')
    end
  end
end

Best practices

  1. Note that this type of tests do not run within the transaction, we use a deletion database cleanup strategy. Do not depend on transaction being present.