Merge branch '29130-api-project-export' into 'master'

Resolve "API endpoint for exporting project"

Closes #29130

See merge request gitlab-org/gitlab-ce!15860
This commit is contained in:
Sean McGivern 2018-03-06 15:24:14 +00:00
commit 39b393fa72
26 changed files with 519 additions and 25 deletions

View file

@ -1527,16 +1527,34 @@ class Project < ActiveRecord::Base
end
end
def import_export_shared
@import_export_shared ||= Gitlab::ImportExport::Shared.new(self)
end
def export_path
return nil unless namespace.present? || hashed_storage?(:repository)
File.join(Gitlab::ImportExport.storage_path, disk_path)
import_export_shared.archive_path
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def export_status
if export_in_progress?
:started
elsif export_project_path
:finished
else
:none
end
end
def export_in_progress?
import_export_shared.active_export_count > 0
end
def remove_exports
return nil unless export_path.present?

View file

@ -2,7 +2,7 @@ module Projects
module ImportExport
class ExportService < BaseService
def execute(_options = {})
@shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.disk_path, 'work'))
@shared = project.import_export_shared
save_all
end

View file

@ -0,0 +1,5 @@
---
title: Add project export API
merge_request: 15860
author: Travis Miller
type: added

View file

@ -1,9 +1,89 @@
# Project import API
# Project import/export API
[Introduced][ce-41899] in GitLab 10.6
[See also the project import/export documentation](../user/project/settings/import_export.md)
## Schedule an export
Start a new export.
```http
POST /projects/:id/export
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
```console
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
```
```json
{
"message": "202 Accepted"
}
```
## Export status
Get the status of export.
```http
GET /projects/:id/export
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
```console
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export
```
Status can be one of `none`, `started`, or `finished`.
`_links` are only present when export has finished.
```json
{
"id": 1,
"description": "Itaque perspiciatis minima aspernatur corporis consequatur.",
"name": "Gitlab Test",
"name_with_namespace": "Gitlab Org / Gitlab Test",
"path": "gitlab-test",
"path_with_namespace": "gitlab-org/gitlab-test",
"created_at": "2017-08-29T04:36:44.383Z",
"export_status": "finished",
"_links": {
"api_url": "https://gitlab.example.com/api/v4/projects/1/export/download",
"web_url": "https://gitlab.example.com/gitlab-org/gitlab-test/download_export",
}
}
```
## Export download
Download the finished export.
```http
GET /projects/:id/export/download
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
```console
curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --remote-header-name --remote-name https://gitlab.example.com/api/v4/projects/5/export/download
```
```console
ls *export.tar.gz
2017-12-05_22-11-148_namespace_project_export.tar.gz
```
## Import a file
```http

View file

@ -139,6 +139,7 @@ module API
mount ::API::PagesDomains
mount ::API::Pipelines
mount ::API::PipelineSchedules
mount ::API::ProjectExport
mount ::API::ProjectImport
mount ::API::ProjectHooks
mount ::API::Projects

View file

@ -91,6 +91,21 @@ module API
expose :created_at
end
class ProjectExportStatus < ProjectIdentity
include ::API::Helpers::RelatedResourcesHelpers
expose :export_status
expose :_links, if: lambda { |project, _options| project.export_status == :finished } do
expose :api_url do |project|
expose_url(api_v4_projects_export_download_path(id: project.id))
end
expose :web_url do |project|
Gitlab::Routing.url_helpers.download_export_project_url(project)
end
end
end
class ProjectImportStatus < ProjectIdentity
expose :import_status

41
lib/api/project_export.rb Normal file
View file

@ -0,0 +1,41 @@
module API
class ProjectExport < Grape::API
before do
not_found! unless Gitlab::CurrentSettings.project_export_enabled?
authorize_admin_project
end
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: { id: %r{[^/]+} } do
desc 'Get export status' do
detail 'This feature was introduced in GitLab 10.6.'
success Entities::ProjectExportStatus
end
get ':id/export' do
present user_project, with: Entities::ProjectExportStatus
end
desc 'Download export' do
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
path = user_project.export_project_path
render_api_error!('404 Not found or has expired', 404) unless path
present_file!(path, File.basename(path), 'application/gzip')
end
desc 'Start export' do
detail 'This feature was introduced in GitLab 10.6.'
end
post ':id/export' do
user_project.add_export_job(current_user: current_user)
accepted!
end
end
end
end

View file

@ -9,7 +9,7 @@ module Gitlab
@archive_file = project.import_source
@current_user = project.creator
@project = project
@shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace)
@shared = project.import_export_shared
end
def execute

View file

@ -1,13 +1,17 @@
module Gitlab
module ImportExport
class Shared
attr_reader :errors, :opts
attr_reader :errors, :project
def initialize(opts)
@opts = opts
def initialize(project)
@project = project
@errors = []
end
def active_export_count
Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) }
end
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
@ -31,11 +35,11 @@ module Gitlab
private
def relative_path
File.join(opts[:relative_path], SecureRandom.hex)
File.join(relative_archive_path, SecureRandom.hex)
end
def relative_archive_path
File.join(opts[:relative_path], '..')
@project.disk_path
end
def error_out(message, caller)

View file

@ -0,0 +1,17 @@
{
"type": "object",
"allOf": [
{ "$ref": "identity.json" },
{
"required": [
"export_status"
],
"properties": {
"export_status": {
"type": "string",
"enum": ["none", "started", "finished"]
}
}
}
]
}

View file

@ -0,0 +1,21 @@
{
"type": "object",
"required": [
"id",
"description",
"name",
"name_with_namespace",
"path",
"path_with_namespace",
"created_at"
],
"properties": {
"id": { "type": "integer" },
"description": { "type": ["string", "null"] },
"name": { "type": "string" },
"name_with_namespace": { "type": "string" },
"path": { "type": "string" },
"path_with_namespace": { "type": "string" },
"created_at": { "type": "date" }
}
}

View file

@ -3,7 +3,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::AvatarRestorer do
include UploadHelpers
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: 'test') }
let(:shared) { project.import_export_shared }
let(:project) { create(:project) }
before do

View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::AvatarSaver do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: 'test') }
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project_with_avatar) { create(:project, avatar: fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png")) }
let(:project) { create(:project) }

View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::FileImporter do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: 'test') }
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" }
let(:valid_file) { "#{shared.export_path}/valid.json" }
let(:symlink_file) { "#{shared.export_path}/invalid.json" }
@ -12,6 +12,7 @@ describe Gitlab::ImportExport::FileImporter do
stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
allow(SecureRandom).to receive(:hex).and_return('abcd')
setup_files
end

View file

@ -7,7 +7,7 @@ describe 'forked project import' do
let!(:project_with_repo) { create(:project, :repository, name: 'test-repo-restorer', path: 'test-repo-restorer') }
let!(:project) { create(:project, name: 'test-repo-restorer-no-repo', path: 'test-repo-restorer-no-repo') }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:forked_from_project) { create(:project, :repository) }
let(:forked_project) { fork_project(project_with_repo, nil, repository: true) }
let(:repo_saver) { Gitlab::ImportExport::RepoSaver.new(project: project_with_repo, shared: shared) }

View file

@ -7,9 +7,9 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
@user = create(:user)
RSpec::Mocks.with_temporary_scope do
@shared = Gitlab::ImportExport::Shared.new(relative_path: "", project_path: 'path')
allow(@shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/')
@project = create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project')
@shared = @project.import_export_shared
allow(@shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/')
allow_any_instance_of(Repository).to receive(:fetch_ref).and_return(true)
allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
@ -263,7 +263,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
context 'Light JSON' do
let(:user) { create(:user) }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "", project_path: 'path') }
let(:shared) { project.import_export_shared }
let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
let(:restored_project_json) { project_tree_restorer.restore }

View file

@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeSaver do
describe 'saves the project tree into a json object' do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:user) { create(:user) }

View file

@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::Reader do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: '') }
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:test_config) { 'spec/support/import_export/import_export.yml' }
let(:project_tree_hash) do
{

View file

@ -6,7 +6,7 @@ describe Gitlab::ImportExport::RepoRestorer do
let!(:project_with_repo) { create(:project, :repository, name: 'test-repo-restorer', path: 'test-repo-restorer') }
let!(:project) { create(:project) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:bundler) { Gitlab::ImportExport::RepoSaver.new(project: project_with_repo, shared: shared) }
let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) }
let(:restorer) do

View file

@ -5,7 +5,7 @@ describe Gitlab::ImportExport::RepoSaver do
let(:user) { create(:user) }
let!(:project) { create(:project, :public, name: 'searchable_project') }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(project: project, shared: shared) }
before do

View file

@ -3,7 +3,7 @@ require 'spec_helper'
describe Gitlab::ImportExport::UploadsRestorer do
describe 'bundle a project Git repo' do
let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)

View file

@ -4,7 +4,7 @@ describe Gitlab::ImportExport::UploadsSaver do
describe 'bundle a project Git repo' do
let(:export_path) { "#{Dir.tmpdir}/uploads_saver_spec" }
let(:file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)

View file

@ -2,12 +2,13 @@ require 'spec_helper'
include ImportExport::CommonUtil
describe Gitlab::ImportExport::VersionChecker do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: '') }
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
describe 'bundle a project Git repo' do
let(:version) { Gitlab::ImportExport.version }
before do
allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('')
allow(File).to receive(:open).and_return(version)
end

View file

@ -5,7 +5,7 @@ describe Gitlab::ImportExport::WikiRepoSaver do
let(:user) { create(:user) }
let!(:project) { create(:project, :public, name: 'searchable_project') }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:wiki_bundler) { described_class.new(project: project, shared: shared) }
let!(:project_wiki) { ProjectWiki.new(project, user) }

View file

@ -6,7 +6,7 @@ describe Gitlab::ImportExport::WikiRestorer do
let!(:project_without_wiki) { create(:project) }
let!(:project) { create(:project) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.full_path) }
let(:shared) { project.import_export_shared }
let(:bundler) { Gitlab::ImportExport::WikiRepoSaver.new(project: project_with_wiki, shared: shared) }
let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) }
let(:restorer) do

View file

@ -0,0 +1,290 @@
require 'spec_helper'
describe API::ProjectExport do
set(:project) { create(:project) }
set(:project_none) { create(:project) }
set(:project_started) { create(:project) }
set(:project_finished) { create(:project) }
set(:user) { create(:user) }
set(:admin) { create(:admin) }
let(:path) { "/projects/#{project.id}/export" }
let(:path_none) { "/projects/#{project_none.id}/export" }
let(:path_started) { "/projects/#{project_started.id}/export" }
let(:path_finished) { "/projects/#{project_finished.id}/export" }
let(:download_path) { "/projects/#{project.id}/export/download" }
let(:download_path_none) { "/projects/#{project_none.id}/export/download" }
let(:download_path_started) { "/projects/#{project_started.id}/export/download" }
let(:download_path_finished) { "/projects/#{project_finished.id}/export/download" }
let(:export_path) { "#{Dir.tmpdir}/project_export_spec" }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
# simulate exporting work directory
FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex')
# simulate exported
FileUtils.mkdir_p project_finished.export_path
FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz')
end
after do
FileUtils.rm_rf(export_path, secure: true)
end
shared_examples_for 'when project export is disabled' do
before do
stub_application_setting(project_export_enabled?: false)
end
it_behaves_like '404 response'
end
describe 'GET /projects/:project_id/export' do
shared_examples_for 'get project export status not found' do
it_behaves_like '404 response' do
let(:request) { get api(path, user) }
end
end
shared_examples_for 'get project export status denied' do
it_behaves_like '403 response' do
let(:request) { get api(path, user) }
end
end
shared_examples_for 'get project export status ok' do
it 'is none' do
get api(path_none, user)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/project/export_status')
expect(json_response['export_status']).to eq('none')
end
it 'is started' do
get api(path_started, user)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/project/export_status')
expect(json_response['export_status']).to eq('started')
end
it 'is finished' do
get api(path_finished, user)
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('public_api/v4/project/export_status')
expect(json_response['export_status']).to eq('finished')
end
end
it_behaves_like 'when project export is disabled' do
let(:request) { get api(path, admin) }
end
context 'when project export is enabled' do
context 'when user is an admin' do
let(:user) { admin }
it_behaves_like 'get project export status ok'
end
context 'when user is a master' do
before do
project.add_master(user)
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
end
it_behaves_like 'get project export status ok'
end
context 'when user is a developer' do
before do
project.add_developer(user)
end
it_behaves_like 'get project export status denied'
end
context 'when user is a reporter' do
before do
project.add_reporter(user)
end
it_behaves_like 'get project export status denied'
end
context 'when user is a guest' do
before do
project.add_guest(user)
end
it_behaves_like 'get project export status denied'
end
context 'when user is not a member' do
it_behaves_like 'get project export status not found'
end
end
end
describe 'GET /projects/:project_id/export/download' do
shared_examples_for 'get project export download not found' do
it_behaves_like '404 response' do
let(:request) { get api(download_path, user) }
end
end
shared_examples_for 'get project export download denied' do
it_behaves_like '403 response' do
let(:request) { get api(download_path, user) }
end
end
shared_examples_for 'get project export download' do
it_behaves_like '404 response' do
let(:request) { get api(download_path_none, user) }
end
it_behaves_like '404 response' do
let(:request) { get api(download_path_started, user) }
end
it 'downloads' do
get api(download_path_finished, user)
expect(response).to have_gitlab_http_status(200)
end
end
it_behaves_like 'when project export is disabled' do
let(:request) { get api(download_path, admin) }
end
context 'when project export is enabled' do
context 'when user is an admin' do
let(:user) { admin }
it_behaves_like 'get project export download'
end
context 'when user is a master' do
before do
project.add_master(user)
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
end
it_behaves_like 'get project export download'
end
context 'when user is a developer' do
before do
project.add_developer(user)
end
it_behaves_like 'get project export download denied'
end
context 'when user is a reporter' do
before do
project.add_reporter(user)
end
it_behaves_like 'get project export download denied'
end
context 'when user is a guest' do
before do
project.add_guest(user)
end
it_behaves_like 'get project export download denied'
end
context 'when user is not a member' do
it_behaves_like 'get project export download not found'
end
end
end
describe 'POST /projects/:project_id/export' do
shared_examples_for 'post project export start not found' do
it_behaves_like '404 response' do
let(:request) { post api(path, user) }
end
end
shared_examples_for 'post project export start denied' do
it_behaves_like '403 response' do
let(:request) { post api(path, user) }
end
end
shared_examples_for 'post project export start' do
it 'starts' do
post api(path, user)
expect(response).to have_gitlab_http_status(202)
end
end
it_behaves_like 'when project export is disabled' do
let(:request) { post api(path, admin) }
end
context 'when project export is enabled' do
context 'when user is an admin' do
let(:user) { admin }
it_behaves_like 'post project export start'
end
context 'when user is a master' do
before do
project.add_master(user)
project_none.add_master(user)
project_started.add_master(user)
project_finished.add_master(user)
end
it_behaves_like 'post project export start'
end
context 'when user is a developer' do
before do
project.add_developer(user)
end
it_behaves_like 'post project export start denied'
end
context 'when user is a reporter' do
before do
project.add_reporter(user)
end
it_behaves_like 'post project export start denied'
end
context 'when user is a guest' do
before do
project.add_guest(user)
end
it_behaves_like 'post project export start denied'
end
context 'when user is not a member' do
it_behaves_like 'post project export start not found'
end
end
end
end