From 9bfa690b7d0a14bdd2791fe62f46dd38f57aa23e Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 9 Feb 2018 15:58:28 +0100 Subject: [PATCH 01/19] add project import spec --- spec/requests/api/project_import_spec.rb | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 spec/requests/api/project_import_spec.rb diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb new file mode 100644 index 00000000000..51980dc1872 --- /dev/null +++ b/spec/requests/api/project_import_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe API::ProjectImport do + let(:export_path) { "#{Dir.tmpdir}/project_export_spec" } + let(:user) { create(:user) } + let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') } + let(:namespace){ create(:group) } + before do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + + group.add_owner(user) + end + + after do + FileUtils.rm_rf(export_path, secure: true) + end + + it 'schedules an import' do + expect_any_instance_of(Project).to receive(:import_schedule) + + post "/projects/import", file: file, namespace: namespace.full_path + + expect(project.status).to eq('started') + end +end From de3edb7178bd7df2f72ef403e57bfdf4e6f732df Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 10:13:08 +0100 Subject: [PATCH 02/19] add more specs --- spec/requests/api/project_import_spec.rb | 34 ++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 51980dc1872..bf7bde0325a 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -4,7 +4,7 @@ describe API::ProjectImport do let(:export_path) { "#{Dir.tmpdir}/project_export_spec" } let(:user) { create(:user) } let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') } - let(:namespace){ create(:group) } + let(:namespace) { create(:group) } before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) @@ -15,11 +15,35 @@ describe API::ProjectImport do FileUtils.rm_rf(export_path, secure: true) end - it 'schedules an import' do - expect_any_instance_of(Project).to receive(:import_schedule) + describe 'POST /projects/import' do - post "/projects/import", file: file, namespace: namespace.full_path + it 'schedules an import' do + expect_any_instance_of(Project).to receive(:import_schedule) - expect(project.status).to eq('started') + post api('/projects/import', user), file: file, namespace: namespace.full_path + + expect(project.status).to eq('started') + end + end + + describe 'GET /projects/:id/import' do + it 'returns the import status' do + project = create(:project, import_status: 'started') + + get api("/projects/#{project.id}/import", user) + + expect(response).to have_gitlab_http_status(200) + expect(json_response).to eq('import_status' => 'started') + end + + it 'returns the import status and the error if failed' do + project = create(:project, import_status: 'failed', import_error: 'error') + + get api("/projects/#{project.id}/import", user) + + expect(response).to have_gitlab_http_status(200) + expect(json_response).to eq('import_status' => 'failed', + 'import_error' => 'error') + end end end From 848f49801d2c45227c525fb5ecdf8b71e7711ded Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 12:40:55 +0100 Subject: [PATCH 03/19] add entity and update spec --- lib/api/api.rb | 1 + lib/api/entities.rb | 5 +++++ spec/requests/api/project_import_spec.rb | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/api/api.rb b/lib/api/api.rb index e953f3d2eca..754549f72f0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -138,6 +138,7 @@ module API mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectImport mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7838de13c56..8fbad2b6959 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -90,6 +90,11 @@ module API expose :created_at end + class ProjectImportStatus < ProjectIdentity + expose :import_status + expose :import_error, if: :import_error + end + class BasicProjectDetails < ProjectIdentity include ::API::ProjectsRelationBuilder diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index bf7bde0325a..55df2d34419 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -33,7 +33,7 @@ describe API::ProjectImport do get api("/projects/#{project.id}/import", user) expect(response).to have_gitlab_http_status(200) - expect(json_response).to eq('import_status' => 'started') + expect(json_response).to include('import_status' => 'started') end it 'returns the import status and the error if failed' do @@ -42,7 +42,7 @@ describe API::ProjectImport do get api("/projects/#{project.id}/import", user) expect(response).to have_gitlab_http_status(200) - expect(json_response).to eq('import_status' => 'failed', + expect(json_response).to include('import_status' => 'failed', 'import_error' => 'error') end end From 82ff66ef31d6ff8ba2332f51b38f79b3bd7d64a5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 12:42:33 +0100 Subject: [PATCH 04/19] add post import API endpoint --- lib/api/project_import.rb | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 lib/api/project_import.rb diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb new file mode 100644 index 00000000000..4bbb78a62f9 --- /dev/null +++ b/lib/api/project_import.rb @@ -0,0 +1,48 @@ +module API + class ProjectImport < Grape::API + include PaginationParams + + helpers do + def import_params + declared_params(include_missing: false) + end + + def file_is_valid? + import_params[:file] && import_params[:file].respond_to?(:read) + end + end + + before do + not_found! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') + end + + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' + requires :file, type: File, desc: 'The project export file to be imported' + end + resource :projects do + desc 'Get export status' do + success Entities::ProjectImportStatus + end + post 'import' do + render_api_error!('The branch refname is invalid', 400) unless file_is_valid? + + namespace = import_params[:namespace] + + namespace = if namespace && namespace =~ /^\d+$/ + Namespace.find_by(id: namespace) + elsif namespace.blank? + current_user.namespace + else + Namespace.find_by_path_or_name(namespace) + end + + project = ::Projects::GitlabProjectsImportService.new(current_user, import_params).execute + + render_api_error!(link.project.full_messages.first, 400) unless project.saved? + + present project, with: Entities::ProjectImportStatus + end + end + end +end From d3b3f5d1b4def5e87f90a2347acc0b0ee8edc80a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 14:46:47 +0100 Subject: [PATCH 05/19] update import API and spec --- lib/api/project_import.rb | 7 +++++-- spec/requests/api/project_import_spec.rb | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 4bbb78a62f9..396c316af22 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -17,6 +17,7 @@ module API end params do + requires :name, type: String, desc: 'The new project name' optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' requires :file, type: File, desc: 'The project export file to be imported' end @@ -37,9 +38,11 @@ module API Namespace.find_by_path_or_name(namespace) end - project = ::Projects::GitlabProjectsImportService.new(current_user, import_params).execute + project_params = import_params.merge(namespace: namespace.id) - render_api_error!(link.project.full_messages.first, 400) unless project.saved? + project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute + + render_api_error!(project.full_messages.first, 400) unless project.saved? present project, with: Entities::ProjectImportStatus end diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 55df2d34419..b810c81108c 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -16,11 +16,10 @@ describe API::ProjectImport do end describe 'POST /projects/import' do - it 'schedules an import' do expect_any_instance_of(Project).to receive(:import_schedule) - post api('/projects/import', user), file: file, namespace: namespace.full_path + post api('/projects/import', user), name: 'test', file: file, namespace: namespace.full_path expect(project.status).to eq('started') end @@ -43,7 +42,7 @@ describe API::ProjectImport do expect(response).to have_gitlab_http_status(200) expect(json_response).to include('import_status' => 'failed', - 'import_error' => 'error') + 'import_error' => 'error') end end end From 516d33f5ac8f65d8d69d1e5e88efbf0faabbe0eb Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 15:26:59 +0100 Subject: [PATCH 06/19] update import API and spec --- lib/api/project_import.rb | 4 ++-- spec/requests/api/project_import_spec.rb | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 396c316af22..1b63f4d4d9f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -17,7 +17,7 @@ module API end params do - requires :name, type: String, desc: 'The new project name' + requires :path, type: String, desc: 'The new project path and name' optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' requires :file, type: File, desc: 'The project export file to be imported' end @@ -38,7 +38,7 @@ module API Namespace.find_by_path_or_name(namespace) end - project_params = import_params.merge(namespace: namespace.id) + project_params = import_params.merge(namespace_id: namespace.id) project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index b810c81108c..0875c1cc149 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -8,7 +8,7 @@ describe API::ProjectImport do before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - group.add_owner(user) + namespace.add_owner(user) end after do @@ -19,9 +19,11 @@ describe API::ProjectImport do it 'schedules an import' do expect_any_instance_of(Project).to receive(:import_schedule) - post api('/projects/import', user), name: 'test', file: file, namespace: namespace.full_path + post api('/projects/import', user), path: 'test-import', file: file, namespace: namespace.full_path - expect(project.status).to eq('started') + expect(response).to have_gitlab_http_status(200) + + expect(Project.find_by_name('test-import').first.status).to eq('started') end end From 7ec1a022b79c68dd3232c0abf07d119f0dad808f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 12 Feb 2018 16:02:15 +0100 Subject: [PATCH 07/19] fix file upload --- lib/api/project_import.rb | 7 ++++--- spec/requests/api/project_import_spec.rb | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 1b63f4d4d9f..5a4e4189a58 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -8,7 +8,7 @@ module API end def file_is_valid? - import_params[:file] && import_params[:file].respond_to?(:read) + import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read) end end @@ -26,7 +26,7 @@ module API success Entities::ProjectImportStatus end post 'import' do - render_api_error!('The branch refname is invalid', 400) unless file_is_valid? + render_api_error!('The file is invalid', 400) unless file_is_valid? namespace = import_params[:namespace] @@ -38,7 +38,8 @@ module API Namespace.find_by_path_or_name(namespace) end - project_params = import_params.merge(namespace_id: namespace.id) + project_params = import_params.merge(namespace_id: namespace.id, + file: import_params[:file]['tempfile']) project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 0875c1cc149..3c9a05f93ea 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -19,9 +19,9 @@ describe API::ProjectImport do it 'schedules an import' do expect_any_instance_of(Project).to receive(:import_schedule) - post api('/projects/import', user), path: 'test-import', file: file, namespace: namespace.full_path + post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path - expect(response).to have_gitlab_http_status(200) + expect(response).to have_gitlab_http_status(201) expect(Project.find_by_name('test-import').first.status).to eq('started') end From 583ed0eb94ff938b4986491e27af5f3c97ea6baf Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 09:24:10 +0100 Subject: [PATCH 08/19] add import status endpoint --- lib/api/project_import.rb | 26 ++++++++++++++++-------- spec/requests/api/project_import_spec.rb | 4 ++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 5a4e4189a58..d554d6d12bd 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -16,12 +16,13 @@ module API not_found! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') end - params do - requires :path, type: String, desc: 'The new project path and name' - optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' - requires :file, type: File, desc: 'The project export file to be imported' - end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do + + params do + requires :path, type: String, desc: 'The new project path and name' + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' + requires :file, type: File, desc: 'The project export file to be imported' + end desc 'Get export status' do success Entities::ProjectImportStatus end @@ -40,13 +41,22 @@ module API project_params = import_params.merge(namespace_id: namespace.id, file: import_params[:file]['tempfile']) - project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute - render_api_error!(project.full_messages.first, 400) unless project.saved? + render_api_error!(project&.full_messages&.first, 400) unless project&.saved? present project, with: Entities::ProjectImportStatus end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + desc 'Get export status' do + success Entities::ProjectImportStatus + end + get ':id/import' do + present user_project, with: Entities::ProjectImportStatus + end end end end diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 3c9a05f93ea..cbea0229b09 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -22,14 +22,13 @@ describe API::ProjectImport do post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path expect(response).to have_gitlab_http_status(201) - - expect(Project.find_by_name('test-import').first.status).to eq('started') end end describe 'GET /projects/:id/import' do it 'returns the import status' do project = create(:project, import_status: 'started') + project.add_master(user) get api("/projects/#{project.id}/import", user) @@ -39,6 +38,7 @@ describe API::ProjectImport do it 'returns the import status and the error if failed' do project = create(:project, import_status: 'failed', import_error: 'error') + project.add_master(user) get api("/projects/#{project.id}/import", user) From 4a0d56daacc3f1825c5f9644a0ea63842fa9da7d Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 09:47:47 +0100 Subject: [PATCH 09/19] fix entity --- lib/api/entities.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8fbad2b6959..ffdb06b8094 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -92,7 +92,9 @@ module API class ProjectImportStatus < ProjectIdentity expose :import_status - expose :import_error, if: :import_error + + # TODO: Use `expose_nil` once we upgrade the grape-entity gem + expose :import_error, if: lambda { |status, _ops| status.import_error } end class BasicProjectDetails < ProjectIdentity From 79879145e5cfb3837b3a2f77fb7c35bd1234068c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 10:54:04 +0100 Subject: [PATCH 10/19] add more specs --- lib/api/project_import.rb | 10 +++---- spec/requests/api/project_import_spec.rb | 33 +++++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index d554d6d12bd..1e51c92cbf1 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -17,7 +17,6 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do - params do requires :path, type: String, desc: 'The new project path and name' optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' @@ -30,11 +29,10 @@ module API render_api_error!('The file is invalid', 400) unless file_is_valid? namespace = import_params[:namespace] - - namespace = if namespace && namespace =~ /^\d+$/ - Namespace.find_by(id: namespace) - elsif namespace.blank? + namespace = if namespace.blank? current_user.namespace + elsif namespace =~ /^\d+$/ + Namespace.find_by(id: namespace) else Namespace.find_by_path_or_name(namespace) end @@ -43,7 +41,7 @@ module API file: import_params[:file]['tempfile']) project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute - render_api_error!(project&.full_messages&.first, 400) unless project&.saved? + render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? present project, with: Entities::ProjectImportStatus end diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index cbea0229b09..b7b22e91bbf 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -16,13 +16,44 @@ describe API::ProjectImport do end describe 'POST /projects/import' do - it 'schedules an import' do + it 'schedules an import using a namespace' do expect_any_instance_of(Project).to receive(:import_schedule) + expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path expect(response).to have_gitlab_http_status(201) end + + it 'schedules an import at the user namespace level' do + expect_any_instance_of(Project).to receive(:import_schedule) + expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(user.namespace.id, any_args).and_call_original + + post api('/projects/import', user), path: 'test-import2', file: fixture_file_upload(file) + + expect(response).to have_gitlab_http_status(201) + end + + it 'does not schedule an import if the user has no permission to the namespace' do + expect_any_instance_of(Project).not_to receive(:import_schedule) + + post(api('/projects/import', create(:user)), + path: 'test-import3', + file: fixture_file_upload(file), + namespace: namespace.full_path) + + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']).to eq('Namespace is not valid') + end + + it 'does not schedule an import if the user uploads no valid file' do + expect_any_instance_of(Project).not_to receive(:import_schedule) + + post api('/projects/import', user), path: 'test-import3', file: './random/test' + + expect(response).to have_gitlab_http_status(400) + expect(json_response['error']).to eq('file is invalid') + end end describe 'GET /projects/:id/import' do From de83f29a3506af01af31c6668640fbb2a0dd54a9 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 11:24:14 +0100 Subject: [PATCH 11/19] add more specs --- spec/requests/api/project_import_spec.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index b7b22e91bbf..47c1edc7919 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -20,6 +20,15 @@ describe API::ProjectImport do expect_any_instance_of(Project).to receive(:import_schedule) expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original + post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.id + + expect(response).to have_gitlab_http_status(201) + end + + it 'schedules an import using the namespace path' do + expect_any_instance_of(Project).to receive(:import_schedule) + expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original + post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path expect(response).to have_gitlab_http_status(201) From 17e5ef4b269f4c13d22bd5c10086c6226dc43543 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 15:04:19 +0100 Subject: [PATCH 12/19] add docs and changelog --- ...ndpoint-for-importing-a-project-export.yml | 5 ++ doc/api/project_import_export.md | 72 +++++++++++++++++++ lib/api/project_import.rb | 2 +- 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/41899-api-endpoint-for-importing-a-project-export.yml create mode 100644 doc/api/project_import_export.md diff --git a/changelogs/unreleased/41899-api-endpoint-for-importing-a-project-export.yml b/changelogs/unreleased/41899-api-endpoint-for-importing-a-project-export.yml new file mode 100644 index 00000000000..29ab7cc7cab --- /dev/null +++ b/changelogs/unreleased/41899-api-endpoint-for-importing-a-project-export.yml @@ -0,0 +1,5 @@ +--- +title: API endpoint for importing a project export +merge_request: 17025 +author: +type: added diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md new file mode 100644 index 00000000000..c594db7d7f2 --- /dev/null +++ b/doc/api/project_import_export.md @@ -0,0 +1,72 @@ +# Project import API + +[Introduced][ce-41899] in GitLab 10.6 + +[Project import/export](../user/project/settings/import_export.md) + +## Import a file + +```http +POST /projects/import +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the user's namespace | +| `file` | string | yes | The file to be uploaded | +| `path` | string | yes | Name and path for new project | + +To upload a file from your filesystem, use the `--form` argument. This causes +cURL to post data using the header `Content-Type: multipart/form-data`. +The `file=` parameter must point to a file on your filesystem and be preceded +by `@`. For example: + +```console +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" -F "path=api-project" -F "file=@/path/to/file" https://gitlab.example.com/api/v4/projects/import +``` + +```json +{ + "id": 1, + "description": null, + "name": "api-project", + "name_with_namespace": "Administrator / api-project", + "path": "api-project", + "path_with_namespace": "root/api-project", + "created_at": "2018-02-13T09:05:58.023Z", + "import_status": "scheduled" +} +``` + +## Import status + +Get the status of export. + +```http +GET /projects/:id/import +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | + +```console +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/import +``` + +Status can be one of `none`, `started`, or `finished`. + +```json +{ + "id": 1, + "description": "Itaque perspiciatis minima aspernatur corporis consequatur.", + "name": "Gitlab Test", + "name_with_namespace": "Gitlab Org / Gitlab Test", + "path": "gitlab-test", + "path_with_namespace": "gitlab-org/gitlab-test", + "created_at": "2017-08-29T04:36:44.383Z", + "import_status": "started" +} +``` + +[ce-41899]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41899 diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 1e51c92cbf1..e93c14ddcf5 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -19,8 +19,8 @@ module API resource :projects, requirements: { id: %r{[^/]+} } do params do requires :path, type: String, desc: 'The new project path and name' - optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' requires :file, type: File, desc: 'The project export file to be imported' + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' end desc 'Get export status' do success Entities::ProjectImportStatus From 39122ea40dda9a1c5b353fc671219b51c625f6cb Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 15:12:18 +0100 Subject: [PATCH 13/19] update docs --- doc/api/project_import_export.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index c594db7d7f2..17b9b530be7 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -40,7 +40,7 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" -F "path=api- ## Import status -Get the status of export. +Get the status of an import. ```http GET /projects/:id/import @@ -54,7 +54,9 @@ GET /projects/:id/import curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/import ``` -Status can be one of `none`, `started`, or `finished`. +Status can be one of `none`, `scheduled`, `failed`, `started`, or `finished`. + +If the status is `failed`, it will include the import error message. ```json { From e8813520029f0f07b0cf2d8463337ae09f15b7fe Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 15:18:52 +0100 Subject: [PATCH 14/19] refactor api class --- lib/api/project_import.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index e93c14ddcf5..b0511342b63 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -13,7 +13,7 @@ module API end before do - not_found! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') + forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') end resource :projects, requirements: { id: %r{[^/]+} } do @@ -22,12 +22,14 @@ module API requires :file, type: File, desc: 'The project export file to be imported' optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' end - desc 'Get export status' do + desc 'Create a new project import' do success Entities::ProjectImportStatus end post 'import' do render_api_error!('The file is invalid', 400) unless file_is_valid? + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') + namespace = import_params[:namespace] namespace = if namespace.blank? current_user.namespace @@ -49,7 +51,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - desc 'Get export status' do + desc 'Get a project export status' do success Entities::ProjectImportStatus end get ':id/import' do From 083194908437f129adcb6e36be40af72ad6d308c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 13 Feb 2018 15:35:08 +0100 Subject: [PATCH 15/19] update missing doc links --- doc/api/projects.md | 4 ++++ doc/user/project/settings/import_export.md | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api/projects.md b/doc/api/projects.md index 46f5de5aa0e..08df65b2f66 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1326,6 +1326,10 @@ POST /projects/:id/housekeeping Read more in the [Branches](branches.md) documentation. +## Project Import/Export + +Read more in the [Project export](project_import_export.md) documentation. + ## Project members Read more in the [Project members](members.md) documentation. diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index b8f865679a2..dedf102fc37 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -22,6 +22,7 @@ > in the import side is required to map the users, based on email or username. > Otherwise, a supplementary comment is left to mention the original author and > the MRs, notes or issues will be owned by the importer. +> - Control project Import/Export with the [API](../../../api/project_import_export.md). Existing projects running on any GitLab instance or GitLab.com can be exported with all their related data and be moved into a new GitLab instance. From e613d777b258a4f7070d2b7aaee093901e4b7ed7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 14 Feb 2018 14:46:40 +0100 Subject: [PATCH 16/19] refactor code based on feedback --- lib/api/project_import.rb | 17 ++++++++------ spec/requests/api/project_import_spec.rb | 28 +++++++++++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index b0511342b63..88fe1d2b5f5 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -10,19 +10,24 @@ module API def file_is_valid? import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read) end + + def validate_file! + render_api_error!('The file is invalid', 400) unless file_is_valid? + end end before do forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do params do requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' end desc 'Create a new project import' do + detail 'This feature was introduced in GitLab 10.6.' success Entities::ProjectImportStatus end post 'import' do @@ -30,13 +35,10 @@ module API Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') - namespace = import_params[:namespace] - namespace = if namespace.blank? - current_user.namespace - elsif namespace =~ /^\d+$/ - Namespace.find_by(id: namespace) + namespace = if import_params[:namespace] + find_namespace!(import_params[:namespace]) else - Namespace.find_by_path_or_name(namespace) + current_user.namespace end project_params = import_params.merge(namespace_id: namespace.id, @@ -52,6 +54,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end desc 'Get a project export status' do + detail 'This feature was introduced in GitLab 10.6.' success Entities::ProjectImportStatus end get ':id/import' do diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 47c1edc7919..2fb103c6510 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -17,8 +17,7 @@ describe API::ProjectImport do describe 'POST /projects/import' do it 'schedules an import using a namespace' do - expect_any_instance_of(Project).to receive(:import_schedule) - expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original + stub_import(user.namespace) post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.id @@ -26,8 +25,7 @@ describe API::ProjectImport do end it 'schedules an import using the namespace path' do - expect_any_instance_of(Project).to receive(:import_schedule) - expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original + stub_import(unamespace) post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path @@ -35,14 +33,23 @@ describe API::ProjectImport do end it 'schedules an import at the user namespace level' do - expect_any_instance_of(Project).to receive(:import_schedule) - expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(user.namespace.id, any_args).and_call_original + stub_import(user.namespace) post api('/projects/import', user), path: 'test-import2', file: fixture_file_upload(file) expect(response).to have_gitlab_http_status(201) end + it 'schedules an import at the user namespace level' do + expect_any_instance_of(Project).not_to receive(:import_schedule) + expect(Gitlab::ImportExport::ProjectCreator).not_to receive(:new) + + post api('/projects/import', user), namespace: 'nonexistent', path: 'test-import2', file: fixture_file_upload(file) + + expect(response).to have_gitlab_http_status(404) + expect(json_response['message']).to eq('404 Namespace Not Found') + end + it 'does not schedule an import if the user has no permission to the namespace' do expect_any_instance_of(Project).not_to receive(:import_schedule) @@ -51,8 +58,8 @@ describe API::ProjectImport do file: fixture_file_upload(file), namespace: namespace.full_path) - expect(response).to have_gitlab_http_status(400) - expect(json_response['message']).to eq('Namespace is not valid') + expect(response).to have_gitlab_http_status(404) + expect(json_response['message']).to eq('404 Namespace Not Found') end it 'does not schedule an import if the user uploads no valid file' do @@ -63,6 +70,11 @@ describe API::ProjectImport do expect(response).to have_gitlab_http_status(400) expect(json_response['error']).to eq('file is invalid') end + + def stub_import(namespace) + expect_any_instance_of(Project).to receive(:import_schedule) + expect(Gitlab::ImportExport::ProjectCreator).to receive(:new).with(namespace.id, any_args).and_call_original + end end describe 'GET /projects/:id/import' do From 2dc2bad6233e1e74c74c49af4be838e0a39dcd53 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 14 Feb 2018 14:55:11 +0100 Subject: [PATCH 17/19] refactor code based on feedback --- doc/api/project_import_export.md | 2 +- spec/requests/api/project_import_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 17b9b530be7..c473b94aafe 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -22,7 +22,7 @@ The `file=` parameter must point to a file on your filesystem and be preceded by `@`. For example: ```console -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" -F "path=api-project" -F "file=@/path/to/file" https://gitlab.example.com/api/v4/projects/import +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "path=api-project" --form "file=@/path/to/file" https://gitlab.example.com/api/v4/projects/import ``` ```json diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 2fb103c6510..eabf9095c13 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -17,7 +17,7 @@ describe API::ProjectImport do describe 'POST /projects/import' do it 'schedules an import using a namespace' do - stub_import(user.namespace) + stub_import(namespace) post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.id @@ -25,7 +25,7 @@ describe API::ProjectImport do end it 'schedules an import using the namespace path' do - stub_import(unamespace) + stub_import(namespace) post api('/projects/import', user), path: 'test-import', file: fixture_file_upload(file), namespace: namespace.full_path From 0abd85f919053efa8a03add9ae43ce9ea2d02ae5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 14 Feb 2018 14:59:11 +0100 Subject: [PATCH 18/19] refactor code based on feedback --- lib/api/project_import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 88fe1d2b5f5..a6da9b90fe3 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -31,7 +31,7 @@ module API success Entities::ProjectImportStatus end post 'import' do - render_api_error!('The file is invalid', 400) unless file_is_valid? + validate_file! Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') From 890d7b540b1ffbadcde490a2e1b741bbb1af3cf4 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 16 Feb 2018 14:37:26 +0100 Subject: [PATCH 19/19] update docs --- doc/api/README.md | 1 + doc/api/project_import_export.md | 6 +++--- doc/api/projects.md | 2 +- lib/api/project_import.rb | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/api/README.md b/doc/api/README.md index f226716c3b5..1927489b7bb 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -43,6 +43,7 @@ following locations: - [Pipeline Schedules](pipeline_schedules.md) - [Projects](projects.md) including setting Webhooks - [Project Access Requests](access_requests.md) +- [Project import/export](project_import_export.md) - [Project Members](members.md) - [Project Snippets](project_snippets.md) - [Protected Branches](protected_branches.md) diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index c473b94aafe..e442442c750 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -2,7 +2,7 @@ [Introduced][ce-41899] in GitLab 10.6 -[Project import/export](../user/project/settings/import_export.md) +[See also the project import/export documentation](../user/project/settings/import_export.md) ## Import a file @@ -12,7 +12,7 @@ POST /projects/import | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the user's namespace | +| `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the current user's namespace | | `file` | string | yes | The file to be uploaded | | `path` | string | yes | Name and path for new project | @@ -56,7 +56,7 @@ curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/a Status can be one of `none`, `scheduled`, `failed`, `started`, or `finished`. -If the status is `failed`, it will include the import error message. +If the status is `failed`, it will include the import error message under `import_error`. ```json { diff --git a/doc/api/projects.md b/doc/api/projects.md index 08df65b2f66..2cdfd1d2ca7 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1328,7 +1328,7 @@ Read more in the [Branches](branches.md) documentation. ## Project Import/Export -Read more in the [Project export](project_import_export.md) documentation. +Read more in the [Project import/export](project_import_export.md) documentation. ## Project members diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index a6da9b90fe3..c32e2f26ae3 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -24,7 +24,7 @@ module API params do requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' - optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be imported into. Defaults to the user namespace.' + optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." end desc 'Create a new project import' do detail 'This feature was introduced in GitLab 10.6.'