Merge branch 'fix/gb/fix-import-export-restoring-associations' into 'master'

Fix restoring associations with import/export

Closes #41646

See merge request gitlab-org/gitlab-ce!16221

app/models/ci/build.rb

@@ -79,7 +79,7 @@ module Ci
     before_save :ensure_token
     before_destroy { unscoped_project }
 
-    after_create do |build|
+    after_create unless: :importing? do |build|
       run_after_commit { BuildHooksWorker.perform_async(build.id) }
     end
 

changelogs/unreleased/fix-gb-fix-import-export-restoring-associations.yml

@@ -0,0 +1,6 @@
+---
+title: Fix missing references to pipeline objects when restoring project with import/export
+  feature
+merge_request: 16221
+author:
+type: fixed

doc/user/project/settings/import_export.md

@@ -30,7 +30,8 @@ with all their related data and be moved into a new GitLab instance.
 
 | GitLab version   | Import/Export version |
 | ---------------- | --------------------- |
-| 10.3 to current  | 0.2.1                 |
+| 10.4 to current  | 0.2.2                 |
+| 10.3             | 0.2.1                 |
 | 10.0             | 0.2.0                 |
 | 9.4.0            | 0.1.8                 |
 | 9.2.0            | 0.1.7                 |

lib/gitlab/import_export.rb

@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.1'.freeze
+    VERSION = '0.2.2'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)

lib/gitlab/import_export/import_export.yml

@@ -49,7 +49,7 @@ project_tree:
       - :author
       - events:
         - :push_event_payload
-    - :stages
+    - stages:
       - :statuses
   - :auto_devops
   - :triggers

lib/gitlab/import_export/relation_factory.rb

@@ -62,6 +62,7 @@ module Gitlab
         when :notes                          then setup_note
         when :project_label, :project_labels then setup_label
         when :milestone, :milestones         then setup_milestone
+        when 'Ci::Pipeline'                  then setup_pipeline
         else
           @relation_hash['project_id'] = @project.id
         end
@@ -112,9 +113,7 @@ module Gitlab
           @relation_hash.delete('trace') # old export files have trace
           @relation_hash.delete('token')
 
-          imported_object do |object|
-            object.commit_id = nil
-          end
+          imported_object
         elsif @relation_name == :merge_requests
           MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
         else
@@ -182,8 +181,9 @@ module Gitlab
       end
 
       def imported_object
-        yield(existing_or_new_object) if block_given?
-        existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
+        if existing_or_new_object.respond_to?(:importing)
+          existing_or_new_object.importing = true
+        end
 
         existing_or_new_object
       rescue ActiveRecord::RecordNotUnique
@@ -211,6 +211,14 @@ module Gitlab
         @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
       end
 
+      def setup_pipeline
+        @relation_hash.fetch('stages').each do |stage|
+          stage.statuses.each do |status|
+            status.pipeline = imported_object
+          end
+        end
+      end
+
       def existing_or_new_object
         # Only find existing records to avoid mapping tables such as milestones
         # Otherwise always create the record, skipping the extra SELECT clause.

spec/lib/gitlab/import_export/project.json

@@ -6465,6 +6465,15 @@
           }
         }
       ],
+      "stages": [
+        {
+          "id": 11,
+          "project_id": 5,
+          "pipeline_id": 36,
+          "name": "test",
+          "status": 1,
+          "created_at": "2016-03-22T15:44:44.772Z",
+          "updated_at": "2016-03-29T06:44:44.634Z",
           "statuses": [
             {
               "id": 71,
@@ -6487,6 +6496,7 @@
               "stage": "test",
               "trigger_request_id": null,
               "stage_idx": 1,
+              "stage_id": 11,
               "tag": null,
               "ref": "master",
               "user_id": null,
@@ -6515,15 +6525,16 @@
               "runner_id": null,
               "coverage": null,
               "commit_id": 36,
-          "commands": "$ build command",
+              "commands": "$ deploy command",
               "job_id": null,
               "name": "test build 2",
               "deploy": false,
               "options": null,
               "allow_failure": false,
-          "stage": "test",
+              "stage": "deploy",
               "trigger_request_id": null,
               "stage_idx": 1,
+              "stage_id": 12,
               "tag": null,
               "ref": "master",
               "user_id": null,
@@ -6540,6 +6551,17 @@
             }
           ]
         },
+        {
+          "id": 12,
+          "project_id": 5,
+          "pipeline_id": 36,
+          "name": "deploy",
+          "status": 2,
+          "created_at": "2016-03-22T15:45:45.772Z",
+          "updated_at": "2016-03-29T06:45:45.634Z"
+        }
+      ]
+    },
     {
       "id": 37,
       "project_id": 5,
@@ -6556,6 +6578,15 @@
       "started_at": null,
       "finished_at": null,
       "duration": null,
+      "stages": [
+        {
+          "id": 21,
+          "project_id": 5,
+          "pipeline_id": 37,
+          "name": "test",
+          "status": 1,
+          "created_at": "2016-03-22T15:44:44.772Z",
+          "updated_at": "2016-03-29T06:44:44.634Z",
           "statuses": [
             {
               "id": 74,
@@ -6628,6 +6659,8 @@
               "erased_at": null
             }
           ]
+        }
+      ]
     },
     {
       "id": 38,
@@ -6645,6 +6678,15 @@
       "started_at": null,
       "finished_at": null,
       "duration": null,
+      "stages": [
+        {
+          "id": 22,
+          "project_id": 5,
+          "pipeline_id": 38,
+          "name": "test",
+          "status": 1,
+          "created_at": "2016-03-22T15:44:44.772Z",
+          "updated_at": "2016-03-29T06:44:44.634Z",
           "statuses": [
             {
               "id": 76,
@@ -6717,6 +6759,8 @@
               "erased_at": null
             }
           ]
+        }
+      ]
     },
     {
       "id": 39,
@@ -6734,6 +6778,15 @@
       "started_at": null,
       "finished_at": null,
       "duration": null,
+      "stages": [
+        {
+          "id": 23,
+          "project_id": 5,
+          "pipeline_id": 39,
+          "name": "test",
+          "status": 1,
+          "created_at": "2016-03-22T15:44:44.772Z",
+          "updated_at": "2016-03-29T06:44:44.634Z",
           "statuses": [
             {
               "id": 78,
@@ -6806,6 +6859,8 @@
               "erased_at": null
             }
           ]
+        }
+      ]
     },
     {
       "id": 40,
@@ -6823,6 +6878,15 @@
       "started_at": null,
       "finished_at": null,
       "duration": null,
+      "stages": [
+        {
+          "id": 24,
+          "project_id": 5,
+          "pipeline_id": 40,
+          "name": "test",
+          "status": 1,
+          "created_at": "2016-03-22T15:44:44.772Z",
+          "updated_at": "2016-03-29T06:44:44.634Z",
           "statuses": [
             {
               "id": 79,
@@ -6896,6 +6960,8 @@
             }
           ]
         }
+      ]
+    }
   ],
   "triggers": [
     {

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -179,6 +179,32 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
             end
         end
       end
+
+      context 'when restoring hierarchy of pipeline, stages and jobs' do
+        it 'restores pipelines' do
+          expect(Ci::Pipeline.all.count).to be 5
+        end
+
+        it 'restores pipeline stages' do
+          expect(Ci::Stage.all.count).to be 6
+        end
+
+        it 'correctly restores association between stage and a pipeline' do
+          expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0))
+        end
+
+        it 'restores statuses' do
+          expect(CommitStatus.all.count).to be 10
+        end
+
+        it 'correctly restores association between a stage and a job' do
+          expect(CommitStatus.all).to all(have_attributes(stage_id: a_value > 0))
+        end
+
+        it 'correctly restores association between a pipeline and a job' do
+          expect(CommitStatus.all).to all(have_attributes(pipeline_id: a_value > 0))
+        end
+      end
     end
   end
 

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

@@ -109,12 +109,20 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
         expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty
       end
 
+      it 'has pipeline stages' do
+        expect(saved_project_json.dig('pipelines', 0, 'stages')).not_to be_empty
+      end
+
       it 'has pipeline statuses' do
-        expect(saved_project_json['pipelines'].first['statuses']).not_to be_empty
+        expect(saved_project_json.dig('pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty
       end
 
       it 'has pipeline builds' do
-        expect(saved_project_json['pipelines'].first['statuses'].count { |hash| hash['type'] == 'Ci::Build' }).to eq(1)
+        builds_count = saved_project_json
+          .dig('pipelines', 0, 'stages', 0, 'statuses')
+          .count { |hash| hash['type'] == 'Ci::Build' }
+
+        expect(builds_count).to eq(1)
       end
 
       it 'has no when YML attributes but only the DB column' do