Squashed commit of the following:

commit 8ea114e5c349ad23f7293cf8141798aa6c9384e3
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Aug 17 13:32:14 2018 +0900

    Remove redandant parentheses

commit 3eb34f9305980701f0c1528f95fd615c1ca4d50d
Merge: 24308e4d1da 46494f46a1
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Aug 17 13:22:05 2018 +0900

    Merge branch 'master-ce' into add-background-migration-for-legacy-traces

commit 24308e4d1dab6fbf8437afc56fff7b83cc7ce534
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 16:39:20 2018 +0900

    Fix spec

commit 5a63312d1b36296440da6a874bb667b7dc06869c
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:46:59 2018 +0900

    Format SQL in MIgrateLegacyArtifacts class

commit 8894196d2aa2f9bf153cb03fef9603858478db70
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:31:18 2018 +0900

    Remove unnecessary index

commit d9753ac058fc48c8f594ee90064f675f7a869b17
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:24:20 2018 +0900

    Decouple pertial index to a different migration file

commit 1ce4a9bec6a4f0f981f4139c1a146ca7d887e6ba
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:14:03 2018 +0900

    Simplified adding file_location column

commit 47b101fae914470dc4d048f6d33c509ec8270656
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:11:42 2018 +0900

    Remove add_column_with_default

commit 80ec4f774f50bbf66e65f283f8fba541c6334e1c
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 14:02:46 2018 +0900

    Simplified job_artifact_uploader

commit d835dab36e85f4fc7a7d0735ede0f0ea081178bf
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 13:59:03 2018 +0900

    Elaborate comments on `file_location`

commit 6e78f23683bbae1372783f19179206369d7fbae7
Merge: fcef07ef361 79fdfec51f
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Aug 16 13:39:28 2018 +0900

    Merge branch 'master-ce' into add-background-migration-for-legacy-traces

commit fcef07ef361080823432071722897321b2dd8354
Merge: 1eac3083194 bf9fd9c3fc
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Jul 6 21:46:22 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 1eac30831947cceec7610be6acc7d3bffadaf617
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Jul 6 21:45:49 2018 +0900

    Use SQL instead of strip_heredoc

commit 08b17e517722e1e4ef8b60e753d53d697953a293
Merge: c0f48ff243d dc478a8243
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Jul 5 23:20:37 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit c0f48ff243ddbe8af6c63aa0170538717191f44a
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Jul 5 17:37:40 2018 +0900

    Fix flaky spec

commit d082dab7cb456f9b31e76f49d1db46b606ba13ce
Merge: 8d6059d08ff 116955c453
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Jul 5 17:35:06 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 8d6059d08fffaa99689afe32b66d93b538fb62f2
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jul 4 21:39:09 2018 +0900

    Fix static analysys failure

commit c4bdb18d582c520723a73c4c90c522332fa94d36
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jul 4 20:16:16 2018 +0900

    Cleanup spec

commit 469e826429feb6a8ce66eb5d5e087f7182eb8813
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jul 4 19:09:12 2018 +0900

    Clean up migrate_legacy_artifacts spec

commit dee84be650ad5afc9f18ac05eccb0736e4b1c121
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jul 4 18:13:19 2018 +0900

    Set default value at the database level

commit b32cb2f8a7e03238cf97d90f9e05ca11b5c8b687
Merge: 83f7c970ad8 4c1a2a9b99
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jul 4 17:06:02 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 83f7c970ad821f1b52999398070facf59f320479
Merge: 823629c1d9a 275fbf24b1
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jul 3 18:39:36 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 823629c1d9aac5e7dcfb6bc393557a72ca3370b4
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jul 2 14:59:54 2018 +0900

    Decouple background migrations of filling nullified file_locations

commit 42ec60e8a74478c62962f5c66abfbe3e908d9e49
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jul 2 14:46:38 2018 +0900

    Unify partial index migrations

commit c145e9306419f704b14e2a92f51a9cf2d105f7e7
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jul 2 14:24:14 2018 +0900

    Make file_location non-null values

commit a442456f6a83327b2736f49b9522084b5675d129
Merge: c2cafa1051c 3cf683629e
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jul 2 13:16:29 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit c2cafa1051cb7a4582c93b653e0ef3506ee11af4
Merge: 8e5f4f88410 e38db19659
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Jun 28 17:13:48 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 8e5f4f884107c0f574545f043fbacea2698ef3a4
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 27 18:54:03 2018 +0900

    Remove unneccesary specs

commit 86c1c68b1000770fcf8086fa71801b332fb6df5a
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 27 18:05:36 2018 +0900

    Add a partial index on ci_builds.id for legacy artifacts search

commit 70aa08c89bb6cd9b12d38b64cbf1838fb919c30f
Merge: 37801122986 292cf66890
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 27 17:15:33 2018 +0900

    Merge branch 'master' into add-background-migration-for-legacy-traces

commit 378011229864c1e056cf995444f947f6b352172c
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu Jun 7 14:18:00 2018 +0900

    Clean up migration code. Defining migration custom class in  only post migration file which requires it for each_batch

commit 5cfe73318b91f377897e9fc50d67145093846459
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 23:57:03 2018 +0900

    Remove indexing for mysql. with_legacy_artifacts targets all archive rows. Enhance tests.

commit 696c030d4a1ae2e1ee5e0eac771e574191fcb477
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 19:59:35 2018 +0900

    Fix static analysys. Split methods. Use `file_types`

commit 86217c650c99d86d5b329229799f250f2ac5fbf6
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 18:55:15 2018 +0900

    Use raw sql for better readability

commit df449404cb167edd61af1e59ca24f25db0a9ca9f
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 17:47:39 2018 +0900

    Revert unneccesary change

commit 764977a1d98b34e34fcd229fca72c9595d62861a
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 16:09:14 2018 +0900

    Remove AddIndexToCiBuildsArtifactsFile. Add temporary index in background migration class.

commit 7728ab3f9dba0af21033e1ca3b0213cacaef85d8
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 15:28:50 2018 +0900

    Try to explicitly separate with postgresql and mysql for addin index

commit b95b47b965413f6590bd7eb38e34b4ef88cd2002
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 14:29:18 2018 +0900

    Dry up code by referring `BackgroundMigration` namespace. Use `BETWEEN` raw SQL to make the edge case explicit.

commit 5b404e46e2a7c0d5a49b9745a8d985a80c9e380a
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 14:15:37 2018 +0900

    Use `store_path` to make spec more explicit

commit 9f80de8e5aed49aae148278b0d27273e92ddbf25
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 14:03:26 2018 +0900

    Use length not limit

commit f78503f3e55cc08d78274767b9381631201262e0
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Wed Jun 6 13:48:51 2018 +0900

    Add length back for indexing

commit b96a57d5e5c7d2a97a1926ec95d67f0a2a1b8c93
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 12:42:14 2018 +0000

    Update schema.rb

commit 9c25acc9752e5cf87f4062dc9f33fb609e8fb7be
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 12:40:48 2018 +0000

    Add the guard clause in #down method too

commit 2bd5dadbd1b742d15bfb0914ad1be7e2182c1755
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 19:41:08 2018 +0900

    Add a gurad caluse to block adding concurrent index for mysql

commit 9a5e12768ed30f75dd796d7c70f54b5e78aaa746
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 19:20:33 2018 +0900

    Specify length of index in schema.rb manually for resolving mysql errors

commit c0ec68af02706d3c01b20b9f589113a900ff2071
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 17:51:40 2018 +0900

    Separate add_concurrent_index by database type

commit b5f4ed37d9f49719fd8235a7069bb93f3c73c2dc
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 17:30:19 2018 +0900

    Fix length size to use TEXT

commit b38b56056f4de384e32ee9acfbf161397de079fd
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 17:20:35 2018 +0900

    Fix schema version for background migration spec

commit 32c48ef7cba61ceb285c6c7c1f48964a204dff20
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 16:22:00 2018 +0900

    Specify length of indexed column. (This is neccessary for TEXT type)

commit a2e975f2e610ffd5f7f400b29a77d6b337005f1d
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jun 5 15:13:42 2018 +0900

    Fix spec; migrate_legacy_artifacts_to_job_artifacts_spec.rb

commit c78c1e3f127b947e46a007bec2d56e0f3a55d7fa
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jun 4 20:15:23 2018 +0900

    Separate add_concurrent_index migration from background migration

commit ad55e33bc2cd622b19f78ace64a69c18870d62df
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jun 4 19:59:14 2018 +0900

    Revert "Remove index_exists? guard"

    This reverts commit d76807838cf06e023b608c16426174a752a8ccf5.

commit b2693be76fb2d643058f7f0fffd0d75ef13c38f0
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jun 4 17:11:47 2018 +0900

    Remove index_exists? guard

commit a2eb053d3b3bd8c8ad49958d58b7225658214a72
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jun 4 15:41:54 2018 +0900

    Wrap insertion and deletion in a single transaction

commit d4bb2c709ce549eb5d7382c03390bfdf6ec90297
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jun 4 15:08:17 2018 +0900

    Add temporary index to ci_builds.artifacts_file to speed up SELECT query

commit 33d62be9ee72a5ee4a61a7bc793df84ea7e85730
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon May 28 17:31:38 2018 +0900

    Add test for legacy path proxy

commit 57ab71a05b464dc4f477837f237eb15e8e51bb84
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon May 28 15:59:36 2018 +0900

    Optimize queries. Add some tests for filtering logic.

commit 8f24dfaac70bc564973bd81745b80421f1a3e1af
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon May 28 15:30:38 2018 +0900

    Fix static analysis

commit ccc60ddd6076f997f63f46cad814e4ac9e4cbdfc
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu May 24 16:51:59 2018 +0900

    Add changelog

commit fdbdb07a40a8343e8292e618f90658e77cc2115e
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu May 24 16:50:55 2018 +0900

    Add spec for migrations

commit 04886af8724258f9971798adcf3eacd5716f20df
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu May 24 14:48:31 2018 +0900

    Revise comments

commit 55bcbee0576b5ac08b4ed461b85d059e0c670159
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Thu May 24 14:40:29 2018 +0900

    Revise comments

commit db6628675abfe24cc10e682a65bf87ca02504deb
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 17:27:12 2018 +0900

    Add spec base (Not finished)

commit e92dcc392eb4ad5aac06a3510a5879a3f084b9ee
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 17:26:49 2018 +0900

    Added artifacts_metadata_store to remove in background migration

commit 91388399d80b58a831b21e6582d5f5ab29e785c7
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 17:26:20 2018 +0900

    Add a condition to exclude jobs which have job_artifacts already

commit e084ce1181d07d0799714eee058d6c9182bc2fcf
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:51:15 2018 +0900

    Clean up migrate_legacy_artifacts.rb (Fix static analysys)

commit ca4e5d33e27690dc276ba87d08dc24bcf31267f4
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:32:59 2018 +0900

    Remove the culcuration method of metadata size

commit eac4c75be932ae6936702ddfd9202f333f18053f
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:11:16 2018 +0900

    Remove Gitlab::BackgroundMigration:: scope

commit 4a5ca96b628a59eeba25778569e719695ac8b5e8
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:09:09 2018 +0900

    Add limit to file_location column

commit 49ff3d9c98dc145fe3c91cd8ba39d7d71d183fb7
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:05:22 2018 +0900

    Use size.to_i instead of if-condition

commit 526656e6ee5683ec7ea07bd6af3438c3fc32375d
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 15 16:02:57 2018 +0900

    Rename current_path to hashed_path

commit 8e6faca7900cf382a35323b2ed0fabede9bf3bf9
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 8 16:53:43 2018 +0900

    Rename location_1/2 to current/legacy_path

commit 5034543255963a250e076f37c5e42fbf4cb0fd05
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue May 1 21:44:10 2018 +0900

    Rename path_type to file_location. Fix JobArtifactUploader to look up correct file location.

commit 4f7e3503f13d02d7a70e7b68bf740a8d8bb525db
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Apr 27 17:33:15 2018 +0900

    Add background migration for legacy artifacts

Add default_value_for to file_location column

Set file_location explicityly with default_values_for

Add file_location to factory

Remove tests of default_values_for

Set file_location nil as default

app/models/ci/job_artifact.rb

@@ -48,6 +48,20 @@ module Ci
       gzip: 3
     }
 
+    # `file_location` indicates where actual files are stored.
+    # Ideally, actual files should be stored in the same directory, and use the same
+    # convention to generate its path. However, sometimes we can't do so due to backward-compatibility.
+    #
+    # legacy_path ... The actual file is stored at a path consists of a timestamp
+    #                 and raw project/model IDs. Those rows were migrated from
+    #                 `ci_builds.artifacts_file` and `ci_builds.artifacts_metadata`
+    # hashed_path ... The actual file is stored at a path consists of a SHA2 based on the project ID.
+    #                 This is the default value.
+    enum file_location: {
+      legacy_path: 1,
+      hashed_path: 2
+    }
+
     FILE_FORMAT_ADAPTERS = {
       gzip: Gitlab::Ci::Build::Artifacts::GzipFileAdapter
     }.freeze
@@ -72,6 +86,10 @@ module Ci
       [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
     end
 
+    def hashed_path?
+      super || self.file_location.nil?
+    end
+
     def expire_in
       expire_at - Time.now if expire_at
     end
@@ -108,7 +126,7 @@ module Ci
     end
 
     def update_project_statistics_after_destroy
-      update_project_statistics(-self.size)
+      update_project_statistics(-self.size.to_i)
     end
 
     def update_project_statistics(difference)

app/uploaders/job_artifact_uploader.rb

@@ -5,6 +5,7 @@ class JobArtifactUploader < GitlabUploader
   include ObjectStorage::Concern
 
   ObjectNotReadyError = Class.new(StandardError)
+  UnknownFileLocationError = Class.new(StandardError)
 
   storage_options Gitlab.config.artifacts
 
@@ -23,10 +24,22 @@ class JobArtifactUploader < GitlabUploader
   def dynamic_segment
     raise ObjectNotReadyError, 'JobArtifact is not ready' unless model.id
 
-    creation_date = model.created_at.utc.strftime('%Y_%m_%d')
+    if model.hashed_path?
+      hashed_path
+    elsif model.legacy_path?
+      legacy_path
+    else
+      raise UnknownFileLocationError
+    end
+  end
 
+  def hashed_path
     File.join(disk_hash[0..1], disk_hash[2..3], disk_hash,
-              creation_date, model.job_id.to_s, model.id.to_s)
+      model.created_at.utc.strftime('%Y_%m_%d'), model.job_id.to_s, model.id.to_s)
+  end
+
+  def legacy_path
+    File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.job_id.to_s)
   end
 
   def disk_hash

changelogs/unreleased/add-background-migration-for-legacy-traces.yml

@@ -0,0 +1,5 @@
+---
+title: Add background migrations for legacy artifacts
+merge_request: 18615
+author:
+type: performance

db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb

@@ -0,0 +1,9 @@
+class AddFileLocationToCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_job_artifacts, :file_location, :integer, limit: 2
+  end
+end

db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb

@@ -0,0 +1,16 @@
+class AddPartialIndexToCiBuildsArtifactsFile < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'partial_index_ci_builds_on_id_with_legacy_artifacts'.freeze
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, :id, where: "artifacts_file <> ''", name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
+  end
+end

db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb

@@ -0,0 +1,32 @@
+class MigrateLegacyArtifactsToJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateLegacyArtifacts'.freeze
+  BATCH_SIZE = 100
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_builds'
+    self.inheritance_column = :_type_disabled
+
+    scope :with_legacy_artifacts, -> { where("artifacts_file <> ''") }
+  end
+
+  def up
+    MigrateLegacyArtifactsToJobArtifacts::Build
+      .with_legacy_artifacts.tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -341,6 +341,7 @@ ActiveRecord::Schema.define(version: 20180816193530) do
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
   add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
+  add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree
   add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
@@ -396,6 +397,7 @@ ActiveRecord::Schema.define(version: 20180816193530) do
     t.string "file"
     t.binary "file_sha256"
     t.integer "file_format", limit: 2
+    t.integer "file_location", limit: 2
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree

lib/gitlab/background_migration/migrate_legacy_artifacts.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/ClassLength
+
+module Gitlab
+  module BackgroundMigration
+    ##
+    # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts`
+    class MigrateLegacyArtifacts
+      FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
+      ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive']
+      METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata']
+      LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path']
+
+      def perform(start_id, stop_id)
+        ActiveRecord::Base.transaction do
+          insert_archives(start_id, stop_id)
+          insert_metadatas(start_id, stop_id)
+          delete_legacy_artifacts(start_id, stop_id)
+        end
+      end
+
+      private
+
+      def insert_archives(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+              ci_job_artifacts (
+                  project_id,
+                  job_id,
+                  expire_at,
+                  file_location,
+                  created_at,
+                  updated_at,
+                  file,
+                  size,
+                  file_store,
+                  file_type
+              )
+          SELECT
+              project_id,
+              id,
+              artifacts_expire_at,
+              #{LEGACY_PATH_FILE_LOCATION},
+              created_at,
+              created_at,
+              artifacts_file,
+              artifacts_size,
+              COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}),
+              #{ARCHIVE_FILE_TYPE}
+          FROM
+              ci_builds
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+              AND NOT EXISTS (
+                  SELECT
+                      1
+                  FROM
+                      ci_job_artifacts
+                  WHERE
+                      ci_builds.id = ci_job_artifacts.job_id
+                      AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE})
+        SQL
+      end
+
+      def insert_metadatas(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+              ci_job_artifacts (
+                  project_id,
+                  job_id,
+                  expire_at,
+                  file_location,
+                  created_at,
+                  updated_at,
+                  file,
+                  size,
+                  file_store,
+                  file_type
+              )
+          SELECT
+              project_id,
+              id,
+              artifacts_expire_at,
+              #{LEGACY_PATH_FILE_LOCATION},
+              created_at,
+              created_at,
+              artifacts_metadata,
+              NULL,
+              COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}),
+              #{METADATA_FILE_TYPE}
+          FROM
+              ci_builds
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+              AND artifacts_metadata <> ''
+              AND NOT EXISTS (
+                  SELECT
+                      1
+                  FROM
+                      ci_job_artifacts
+                  WHERE
+                      ci_builds.id = ci_job_artifacts.job_id
+                      AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE})
+        SQL
+      end
+
+      def delete_legacy_artifacts(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          UPDATE
+              ci_builds
+          SET
+              artifacts_file = NULL,
+              artifacts_file_store = NULL,
+              artifacts_size = NULL,
+              artifacts_metadata = NULL,
+              artifacts_metadata_store = NULL
+          WHERE
+              id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+              AND artifacts_file <> ''
+        SQL
+      end
+    end
+  end
+end

spec/factories/ci/job_artifacts.rb

@@ -24,6 +24,12 @@ FactoryBot.define do
       end
     end
 
+    trait :legacy_archive do
+      archive
+
+      file_location :legacy_path
+    end
+
     trait :metadata do
       file_type :metadata
       file_format :gzip

spec/lib/gitlab/background_migration/migrate_legacy_artifacts_spec.rb

@@ -0,0 +1,156 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateLegacyArtifacts, :migration, schema: 20180816161409 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:jobs) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+
+  subject { described_class.new.perform(*range) }
+
+  context 'when a pipeline exists' do
+    let!(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+    let!(:project) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
+    let!(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
+
+    context 'when a legacy artifacts exists' do
+      let(:artifacts_expire_at) { 1.day.since.to_s }
+      let(:file_store) { ::ObjectStorage::Store::REMOTE }
+
+      let!(:job) do
+        jobs.create!(
+          commit_id: pipeline.id,
+          project_id: project.id,
+          status: :success,
+          **artifacts_archive_attributes,
+          **artifacts_metadata_attributes)
+      end
+
+      let(:artifacts_archive_attributes) do
+        {
+          artifacts_file: 'archive.zip',
+          artifacts_file_store: file_store,
+          artifacts_size: 123,
+          artifacts_expire_at: artifacts_expire_at
+        }
+      end
+
+      let(:artifacts_metadata_attributes) do
+        {
+          artifacts_metadata: 'metadata.gz',
+          artifacts_metadata_store: file_store
+        }
+      end
+
+      it 'has legacy artifacts' do
+        expect(jobs.pluck('artifacts_file, artifacts_file_store, artifacts_size, artifacts_expire_at')).to eq([artifacts_archive_attributes.values])
+        expect(jobs.pluck('artifacts_metadata, artifacts_metadata_store')).to eq([artifacts_metadata_attributes.values])
+      end
+
+      it 'does not have new artifacts yet' do
+        expect(job_artifacts.count).to be_zero
+      end
+
+      context 'when the record exists inside of the range of a background migration' do
+        let(:range) { [job.id, job.id] }
+
+        it 'migrates a legacy artifact to ci_job_artifacts table' do
+          expect { subject }.to change { job_artifacts.count }.by(2)
+
+          expect(job_artifacts.order(:id).pluck('project_id, job_id, file_type, file_store, size, expire_at, file, file_sha256, file_location'))
+            .to eq([[project.id,
+                     job.id,
+                     described_class::ARCHIVE_FILE_TYPE,
+                     file_store,
+                     artifacts_archive_attributes[:artifacts_size],
+                     artifacts_expire_at,
+                     'archive.zip',
+                     nil,
+                     described_class::LEGACY_PATH_FILE_LOCATION],
+                    [project.id,
+                     job.id,
+                     described_class::METADATA_FILE_TYPE,
+                     file_store,
+                     nil,
+                     artifacts_expire_at,
+                     'metadata.gz',
+                     nil,
+                     described_class::LEGACY_PATH_FILE_LOCATION]])
+
+          expect(jobs.pluck('artifacts_file, artifacts_file_store, artifacts_size, artifacts_expire_at')).to eq([[nil, nil, nil, artifacts_expire_at]])
+          expect(jobs.pluck('artifacts_metadata, artifacts_metadata_store')).to eq([[nil, nil]])
+        end
+
+        context 'when file_store is nil' do
+          let(:file_store) { nil }
+
+          it 'has nullified file_store in all legacy artifacts' do
+            expect(jobs.pluck('artifacts_file_store, artifacts_metadata_store')).to eq([[nil, nil]])
+          end
+
+          it 'fills file_store by the value of local file store' do
+            subject
+
+            expect(job_artifacts.pluck('file_store')).to all(eq(::ObjectStorage::Store::LOCAL))
+          end
+        end
+
+        context 'when new artifacts has already existed' do
+          context 'when only archive.zip existed' do
+            before do
+              job_artifacts.create!(project_id: project.id, job_id: job.id, file_type: described_class::ARCHIVE_FILE_TYPE, size: 999, file: 'archive.zip')
+            end
+
+            it 'had archive.zip already' do
+              expect(job_artifacts.exists?(job_id: job.id, file_type: described_class::ARCHIVE_FILE_TYPE)).to be_truthy
+            end
+
+            it 'migrates metadata' do
+              expect { subject }.to change { job_artifacts.count }.by(1)
+
+              expect(job_artifacts.exists?(job_id: job.id, file_type: described_class::METADATA_FILE_TYPE)).to be_truthy
+            end
+          end
+
+          context 'when both archive and metadata existed' do
+            before do
+              job_artifacts.create!(project_id: project.id, job_id: job.id, file_type: described_class::ARCHIVE_FILE_TYPE, size: 999, file: 'archive.zip')
+              job_artifacts.create!(project_id: project.id, job_id: job.id, file_type: described_class::METADATA_FILE_TYPE, size: 999, file: 'metadata.zip')
+            end
+
+            it 'does not migrate' do
+              expect { subject }.not_to change { job_artifacts.count }
+            end
+          end
+        end
+      end
+
+      context 'when the record exists outside of the range of a background migration' do
+        let(:range) { [job.id + 1, job.id + 1] }
+
+        it 'does not migrate' do
+          expect { subject }.not_to change { job_artifacts.count }
+        end
+      end
+    end
+
+    context 'when the job does not have legacy artifacts' do
+      let!(:job) { jobs.create!(commit_id: pipeline.id, project_id: project.id, status: :success) }
+
+      it 'does not have the legacy artifacts in database' do
+        expect(jobs.count).to eq(1)
+        expect(jobs.pluck('artifacts_file, artifacts_file_store, artifacts_size, artifacts_expire_at')).to eq([[nil, nil, nil, nil]])
+        expect(jobs.pluck('artifacts_metadata, artifacts_metadata_store')).to eq([[nil, nil]])
+      end
+
+      context 'when the record exists inside of the range of a background migration' do
+        let(:range) { [job.id, job.id] }
+
+        it 'does not migrate' do
+          expect { subject }.not_to change { job_artifacts.count }
+        end
+      end
+    end
+  end
+end

spec/migrations/migrate_legacy_artifacts_to_job_artifacts_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb')
+
+describe MigrateLegacyArtifactsToJobArtifacts, :migration, :sidekiq do
+  let(:migration_class) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts }
+  let(:migration_name)  { migration_class.to_s.demodulize }
+
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:jobs) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+  let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let(:project) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
+  let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
+  let(:archive_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::ARCHIVE_FILE_TYPE }
+  let(:metadata_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::METADATA_FILE_TYPE }
+  let(:local_store) { ::ObjectStorage::Store::LOCAL }
+  let(:remote_store) { ::ObjectStorage::Store::REMOTE }
+  let(:legacy_location) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::LEGACY_PATH_FILE_LOCATION }
+
+  context 'when legacy artifacts exist' do
+    before do
+      jobs.create!(id: 1, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip')
+      jobs.create!(id: 2, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 3, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 4, commit_id: pipeline.id, project_id: project.id, status: :running)
+      jobs.create!(id: 5, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip', artifacts_file_store: remote_store, artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 6, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
+    end
+
+    it 'schedules a background migration' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(migration_name).to be_scheduled_delayed_migration(5.minutes, 1, 6)
+          expect(BackgroundMigrationWorker.jobs.size).to eq 1
+        end
+      end
+    end
+
+    it 'migrates legacy artifacts to ci_job_artifacts table' do
+      migrate!
+
+      expect(job_artifacts.order(:job_id, :file_type).pluck('project_id, job_id, file_type, file_store, size, expire_at, file, file_sha256, file_location'))
+        .to eq([[project.id, 1, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 3, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 3, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
+                [project.id, 5, archive_file_type, remote_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 5, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
+                [project.id, 6, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 6, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location]])
+    end
+  end
+
+  context 'when legacy artifacts do not exist' do
+    before do
+      jobs.create!(id: 1, commit_id: pipeline.id, project_id: project.id, status: :success)
+      jobs.create!(id: 2, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_metadata: 'metadata.gz')
+    end
+
+    it 'does not schedule background migrations' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq 0
+        end
+      end
+    end
+  end
+end

spec/uploaders/job_artifact_uploader_spec.rb

@@ -40,6 +40,53 @@ describe JobArtifactUploader do
     it { is_expected.to end_with("ci_build_artifacts.zip") }
   end
 
+  describe '#dynamic_segment' do
+    let(:uploaded_content) { File.binread(Rails.root + 'spec/fixtures/ci_build_artifacts.zip') }
+    let(:model) { uploader.model }
+
+    shared_examples_for 'Read file from legacy path' do
+      it 'store_path returns the legacy path' do
+        expect(model.file.store_path).to eq(File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.job_id.to_s, 'ci_build_artifacts.zip'))
+      end
+
+      it 'has exactly the same content' do
+        expect(::File.binread(model.file.path)).to eq(uploaded_content)
+      end
+    end
+
+    shared_examples_for 'Read file from hashed path' do
+      it 'store_path returns hashed path' do
+        expect(model.file.store_path).to eq(File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, creation_date, model.job_id.to_s, model.id.to_s, 'ci_build_artifacts.zip'))
+      end
+
+      it 'has exactly the same content' do
+        expect(::File.binread(model.file.path)).to eq(uploaded_content)
+      end
+    end
+
+    context 'when a job artifact is stored in legacy_path' do
+      let(:job_artifact) { create(:ci_job_artifact, :legacy_archive) }
+
+      it_behaves_like 'Read file from legacy path'
+    end
+
+    context 'when the artifact file is stored in hashed_path' do
+      let(:job_artifact) { create(:ci_job_artifact, :archive) }
+      let(:disk_hash) { Digest::SHA2.hexdigest(model.project_id.to_s) }
+      let(:creation_date) { model.created_at.utc.strftime('%Y_%m_%d') }
+
+      it_behaves_like 'Read file from hashed path'
+
+      context 'when file_location column is empty' do
+        before do
+          job_artifact.update_column(:file_location, nil)
+        end
+
+        it_behaves_like 'Read file from hashed path'
+      end
+    end
+  end
+
   describe "#migrate!" do
     before do
       uploader.store!(fixture_file_upload('spec/fixtures/trace/sample_trace'))