From 02276f7a70466f08bef2c233b321bee00727b8c2 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Dec 2017 17:21:13 +0100 Subject: [PATCH 01/10] Fix import scheduled on bare repo import --- app/services/projects/create_service.rb | 3 ++- lib/gitlab/bare_repository_import/importer.rb | 2 ++ spec/lib/gitlab/bare_repository_import/importer_spec.rb | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 71533da31b1..b32bbd56529 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -12,6 +12,7 @@ module Projects forked_from_project_id = params.delete(:forked_from_project_id) import_data = params.delete(:import_data) @skip_wiki = params.delete(:skip_wiki) + @skip_import = params.delete(:skip_import) @project = Project.new(params) @@ -57,7 +58,7 @@ module Projects after_create_actions if @project.persisted? if @project.errors.empty? - @project.import_schedule if @project.import? + @project.import_schedule if @project.import? && !@skip_import else fail(error: @project.errors.full_messages.join(', ')) end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 298409d8b5a..b0b349088d9 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -55,6 +55,8 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, + skip_wiki: bare_repo.wiki_exists?, + skip_import: true, import_type: 'gitlab_project', namespace_id: group&.id).execute diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 8a83e446935..920819cf382 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -68,6 +68,12 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do expect(Project.find_by_full_path(project_path)).not_to be_nil end + it 'does not schedule an import' do + expect_any_instance_of(Project).not_to receive(:import_schedule) + + importer.create_project_if_needed + end + it 'creates the Git repo in disk' do FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) @@ -161,6 +167,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git")) + expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_import: true)).and_call_original + importer.create_project_if_needed project = Project.find_by_full_path(project_path) From e77c88e77d59446fdba80869f296424ebf6473c9 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Dec 2017 17:30:56 +0100 Subject: [PATCH 02/10] add changelog --- ...4-gitlab-rake-gitlab-import-repos-schedules-an-import.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml diff --git a/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml b/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml new file mode 100644 index 00000000000..b5f7ab2ce1c --- /dev/null +++ b/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml @@ -0,0 +1,5 @@ +--- +title: Fix gitlab-rake gitlab:import:repos import schedule +merge_request: +author: +type: fixed From 928bf02df02d24fe6607d027cde49b18741a53c0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Dec 2017 16:35:17 +0000 Subject: [PATCH 03/10] Update 41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml --- ...1424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml b/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml index b5f7ab2ce1c..b495754a5a8 100644 --- a/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml +++ b/changelogs/unreleased/41424-gitlab-rake-gitlab-import-repos-schedules-an-import.yml @@ -1,5 +1,5 @@ --- title: Fix gitlab-rake gitlab:import:repos import schedule -merge_request: +merge_request: 16115 author: type: fixed From 9d0bceb3f012aa7a9c412f83dfc775a7f99f4729 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Dec 2017 17:37:43 +0100 Subject: [PATCH 04/10] add extra spec check --- spec/lib/gitlab/bare_repository_import/importer_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 920819cf382..07cc73ee479 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -167,7 +167,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git")) - expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_import: true)).and_call_original + expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true, + skip_import: true)).and_call_original importer.create_project_if_needed From 54f00b74873f7d24fbb6125b0f35fca38fe033b7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 2 Jan 2018 10:38:51 +0100 Subject: [PATCH 05/10] fix wiki check --- lib/gitlab/bare_repository_import/importer.rb | 2 ++ spec/lib/gitlab/bare_repository_import/importer_spec.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index b0b349088d9..3fa2a7e8dfe 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -77,6 +77,8 @@ module Gitlab if bare_repo.wiki_exists? FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + else + project.create_wiki end true diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 07cc73ee479..9fadfadeddd 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -137,6 +137,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do project = Project.find_by_full_path("#{admin.full_path}/#{project_path}") expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.git')) + expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) end it 'moves an existing project to the correct path' do From 3faa988eb8a3098d05045c554ff7540fa5ab29bf Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 2 Jan 2018 15:16:40 +0100 Subject: [PATCH 06/10] fix branch complexity --- app/services/projects/create_service.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index b32bbd56529..6dd51cc2bef 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -57,11 +57,7 @@ module Projects after_create_actions if @project.persisted? - if @project.errors.empty? - @project.import_schedule if @project.import? && !@skip_import - else - fail(error: @project.errors.full_messages.join(', ')) - end + import_schedule @project rescue ActiveRecord::RecordInvalid => e @@ -165,5 +161,15 @@ module Projects @project.path = @project.name.dup.parameterize end end + + private + + def import_schedule + if @project.errors.empty? + @project.import_schedule if @project.import? && !@skip_import + else + fail(error: @project.errors.full_messages.join(', ')) + end + end end end From 11d0083e95b9dc22c27f368e38058ec3e0a40f46 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 2 Jan 2018 16:48:46 +0100 Subject: [PATCH 07/10] fix missing create services call --- app/services/projects/create_service.rb | 2 +- lib/gitlab/bare_repository_import/importer.rb | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 6dd51cc2bef..4ed65982326 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -88,7 +88,7 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") - unless @project.gitlab_project_import? + if !@project.gitlab_project_import? || @skip_import @project.create_wiki unless skip_wiki? create_services_from_active_templates(@project) diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 3fa2a7e8dfe..b0b349088d9 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -77,8 +77,6 @@ module Gitlab if bare_repo.wiki_exists? FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) - else - project.create_wiki end true From f2d4b24a9404a1439d4f4c983fdf9ca000ac6e21 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 3 Jan 2018 09:10:43 +0100 Subject: [PATCH 08/10] refactor code to use new import type --- app/models/project.rb | 4 ++++ app/services/projects/create_service.rb | 7 +++---- lib/gitlab/bare_repository_import/importer.rb | 3 +-- spec/lib/gitlab/bare_repository_import/importer_spec.rb | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 6678733e43e..e37eae9f176 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -679,6 +679,10 @@ class Project < ActiveRecord::Base Gitlab::UrlSanitizer.new(import_url).masked_url end + def bare_repository_import? + import_type == 'bare_repository' + end + def gitlab_project_import? import_type == 'gitlab_project' end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 4ed65982326..52b90cdf135 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -12,7 +12,6 @@ module Projects forked_from_project_id = params.delete(:forked_from_project_id) import_data = params.delete(:import_data) @skip_wiki = params.delete(:skip_wiki) - @skip_import = params.delete(:skip_import) @project = Project.new(params) @@ -88,7 +87,7 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") - if !@project.gitlab_project_import? || @skip_import + unless @project.gitlab_project_import? @project.create_wiki unless skip_wiki? create_services_from_active_templates(@project) @@ -121,7 +120,7 @@ module Projects Project.transaction do @project.create_or_update_import_data(data: import_data[:data], credentials: import_data[:credentials]) if import_data - if @project.save && !@project.import? + if @project.save && !@project.import? && !@project.bare_repository_import? raise 'Failed to create repository' unless @project.create_repository end end @@ -166,7 +165,7 @@ module Projects def import_schedule if @project.errors.empty? - @project.import_schedule if @project.import? && !@skip_import + @project.import_schedule if @project.import? else fail(error: @project.errors.full_messages.join(', ')) end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index b0b349088d9..64e41d42709 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -56,8 +56,7 @@ module Gitlab path: project_name, skip_disk_validation: true, skip_wiki: bare_repo.wiki_exists?, - skip_import: true, - import_type: 'gitlab_project', + import_type: 'bare_repository', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 9fadfadeddd..82ac3c424d4 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -169,7 +169,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git")) expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true, - skip_import: true)).and_call_original + import_type: 'bare_repository')).and_call_original importer.create_project_if_needed From 80112ca4ec0104c06e939a546b92c7bdb230c01f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 3 Jan 2018 09:31:32 +0100 Subject: [PATCH 09/10] add note to import sources --- lib/gitlab/import_sources.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index eeb03625479..60d5fa4d29a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,6 +7,7 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) + # We exclude `bare_repository` here as it has no import class associated ImportTable = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), From 86257cf7138a6d28c055071219142722787b6546 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 3 Jan 2018 16:17:16 +0100 Subject: [PATCH 10/10] refactor project create service --- app/models/project.rb | 2 +- app/services/projects/create_service.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index e37eae9f176..6ebb083aeb4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -639,7 +639,7 @@ class Project < ActiveRecord::Base end def import? - external_import? || forked? || gitlab_project_import? + external_import? || forked? || gitlab_project_import? || bare_repository_import? end def no_import? diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 52b90cdf135..dc7b1f1f5cc 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -120,7 +120,7 @@ module Projects Project.transaction do @project.create_or_update_import_data(data: import_data[:data], credentials: import_data[:credentials]) if import_data - if @project.save && !@project.import? && !@project.bare_repository_import? + if @project.save && !@project.import? raise 'Failed to create repository' unless @project.create_repository end end @@ -165,7 +165,7 @@ module Projects def import_schedule if @project.errors.empty? - @project.import_schedule if @project.import? + @project.import_schedule if @project.import? && !@project.bare_repository_import? else fail(error: @project.errors.full_messages.join(', ')) end