Merge branch 'fj-48123-fix-gitlab-import' into 'master'
Fix gitlab import project load Closes #48123 See merge request gitlab-org/gitlab-ce!20599
This commit is contained in:
commit
054b4f2fc4
|
@ -1,4 +1,7 @@
|
||||||
class Import::GitlabController < Import::BaseController
|
class Import::GitlabController < Import::BaseController
|
||||||
|
MAX_PROJECT_PAGES = 15
|
||||||
|
PER_PAGE_PROJECTS = 100
|
||||||
|
|
||||||
before_action :verify_gitlab_import_enabled
|
before_action :verify_gitlab_import_enabled
|
||||||
before_action :gitlab_auth, except: :callback
|
before_action :gitlab_auth, except: :callback
|
||||||
|
|
||||||
|
@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController
|
||||||
end
|
end
|
||||||
|
|
||||||
def status
|
def status
|
||||||
@repos = client.projects
|
@repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
|
||||||
|
|
||||||
@already_added_projects = find_already_added_projects('gitlab')
|
@already_added_projects = find_already_added_projects('gitlab')
|
||||||
already_added_projects_names = @already_added_projects.pluck(:import_source)
|
already_added_projects_names = @already_added_projects.pluck(:import_source)
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
title: Fix GitLab project imports not loading due to API timeouts
|
||||||
|
merge_request: 20599
|
||||||
|
author:
|
||||||
|
type: fixed
|
|
@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the
|
||||||
"avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png",
|
"avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png",
|
||||||
"star_count":3812,
|
"star_count":3812,
|
||||||
"forks_count":3561,
|
"forks_count":3561,
|
||||||
"last_activity_at":"2018-01-02T11:40:26.570Z"
|
"last_activity_at":"2018-01-02T11:40:26.570Z",
|
||||||
|
"namespace": {
|
||||||
|
"id": 72,
|
||||||
|
"name": "GitLab.org",
|
||||||
|
"path": "gitlab-org",
|
||||||
|
"kind": "group",
|
||||||
|
"full_path": "gitlab-org",
|
||||||
|
"parent_id": null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
...
|
...
|
||||||
|
|
|
@ -132,6 +132,7 @@ module API
|
||||||
expose :star_count, :forks_count
|
expose :star_count, :forks_count
|
||||||
expose :last_activity_at
|
expose :last_activity_at
|
||||||
|
|
||||||
|
expose :namespace, using: 'API::Entities::NamespaceBasic'
|
||||||
expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
|
expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
|
||||||
|
|
||||||
def self.preload_relation(projects_relation, options = {})
|
def self.preload_relation(projects_relation, options = {})
|
||||||
|
@ -194,7 +195,6 @@ module API
|
||||||
expose :shared_runners_enabled
|
expose :shared_runners_enabled
|
||||||
expose :lfs_enabled?, as: :lfs_enabled
|
expose :lfs_enabled?, as: :lfs_enabled
|
||||||
expose :creator_id
|
expose :creator_id
|
||||||
expose :namespace, using: 'API::Entities::NamespaceBasic'
|
|
||||||
expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
|
expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
|
||||||
expose :import_status
|
expose :import_status
|
||||||
expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
|
expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
|
||||||
|
|
|
@ -32,15 +32,15 @@ module Gitlab
|
||||||
api.get("/api/v4/user").parsed
|
api.get("/api/v4/user").parsed
|
||||||
end
|
end
|
||||||
|
|
||||||
def issues(project_identifier)
|
def issues(project_identifier, **kwargs)
|
||||||
lazy_page_iterator(PER_PAGE) do |page|
|
lazy_page_iterator(**kwargs) do |page, per_page|
|
||||||
api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
|
api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def issue_comments(project_identifier, issue_id)
|
def issue_comments(project_identifier, issue_id, **kwargs)
|
||||||
lazy_page_iterator(PER_PAGE) do |page|
|
lazy_page_iterator(**kwargs) do |page, per_page|
|
||||||
api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
|
api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -48,23 +48,27 @@ module Gitlab
|
||||||
api.get("/api/v4/projects/#{id}").parsed
|
api.get("/api/v4/projects/#{id}").parsed
|
||||||
end
|
end
|
||||||
|
|
||||||
def projects
|
def projects(**kwargs)
|
||||||
lazy_page_iterator(PER_PAGE) do |page|
|
lazy_page_iterator(**kwargs) do |page, per_page|
|
||||||
api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
|
api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def lazy_page_iterator(per_page)
|
def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
|
||||||
Enumerator.new do |y|
|
Enumerator.new do |y|
|
||||||
page = 1
|
page = starting_page
|
||||||
|
page_limit = (starting_page - 1) + page_limit if page_limit
|
||||||
|
|
||||||
loop do
|
loop do
|
||||||
items = yield(page)
|
items = yield(page, per_page)
|
||||||
|
|
||||||
items.each do |item|
|
items.each do |item|
|
||||||
y << item
|
y << item
|
||||||
end
|
end
|
||||||
break if items.empty? || items.size < per_page
|
|
||||||
|
break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
|
||||||
|
|
||||||
page += 1
|
page += 1
|
||||||
end
|
end
|
||||||
|
|
|
@ -24,13 +24,24 @@
|
||||||
"avatar_url": { "type": ["string", "null"] },
|
"avatar_url": { "type": ["string", "null"] },
|
||||||
"star_count": { "type": "integer" },
|
"star_count": { "type": "integer" },
|
||||||
"forks_count": { "type": "integer" },
|
"forks_count": { "type": "integer" },
|
||||||
"last_activity_at": { "type": "date" }
|
"last_activity_at": { "type": "date" },
|
||||||
|
"namespace": {
|
||||||
|
"type": "object",
|
||||||
|
"properties" : {
|
||||||
|
"id": { "type": "integer" },
|
||||||
|
"name": { "type": "string" },
|
||||||
|
"path": { "type": "string" },
|
||||||
|
"kind": { "type": "string" },
|
||||||
|
"full_path": { "type": "string" },
|
||||||
|
"parent_id": { "type": ["integer", "null"] }
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
"id", "name", "name_with_namespace", "description", "path",
|
"id", "name", "name_with_namespace", "description", "path",
|
||||||
"path_with_namespace", "created_at", "default_branch", "tag_list",
|
"path_with_namespace", "created_at", "default_branch", "tag_list",
|
||||||
"ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
|
"ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
|
||||||
"star_count", "last_activity_at"
|
"star_count", "last_activity_at", "namespace"
|
||||||
],
|
],
|
||||||
"additionalProperties": false
|
"additionalProperties": false
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do
|
||||||
expect(key).to be_kind_of(Symbol)
|
expect(key).to be_kind_of(Symbol)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it 'uses membership and simple flags' do
|
||||||
|
stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true')
|
||||||
|
|
||||||
|
expect_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
|
||||||
|
|
||||||
|
expect(client.projects.to_a).to eq []
|
||||||
|
end
|
||||||
|
|
||||||
|
shared_examples 'pagination params' do
|
||||||
|
before do
|
||||||
|
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'allows page_limit param' do
|
||||||
|
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return(element_list)
|
||||||
|
|
||||||
|
expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original
|
||||||
|
|
||||||
|
client.send(method, *args, page_limit: 2, per_page: 1).to_a
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'allows per_page param' do
|
||||||
|
expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original
|
||||||
|
|
||||||
|
client.send(method, *args, per_page: 2).to_a
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'allows starting_page param' do
|
||||||
|
expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original
|
||||||
|
|
||||||
|
client.send(method, *args, starting_page: 3).to_a
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#projects' do
|
||||||
|
subject(:method) { :projects }
|
||||||
|
let(:args) { [] }
|
||||||
|
let(:element_list) { build_list(:project, 2) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true')
|
||||||
|
stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true')
|
||||||
|
stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true')
|
||||||
|
stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true')
|
||||||
|
end
|
||||||
|
|
||||||
|
it_behaves_like 'pagination params'
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#issues' do
|
||||||
|
subject(:method) { :issues }
|
||||||
|
let(:args) { [1] }
|
||||||
|
let(:element_list) { build_list(:issue, 2) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
stub_request('/api/v4/projects/1/issues?page=1&per_page=1')
|
||||||
|
stub_request('/api/v4/projects/1/issues?page=2&per_page=1')
|
||||||
|
stub_request('/api/v4/projects/1/issues?page=1&per_page=2')
|
||||||
|
stub_request('/api/v4/projects/1/issues?page=3&per_page=100')
|
||||||
|
end
|
||||||
|
|
||||||
|
it_behaves_like 'pagination params'
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#issue_comments' do
|
||||||
|
subject(:method) { :issue_comments }
|
||||||
|
let(:args) { [1, 1] }
|
||||||
|
let(:element_list) { build_list(:note_on_issue, 2) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1')
|
||||||
|
stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1')
|
||||||
|
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2')
|
||||||
|
stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100')
|
||||||
|
end
|
||||||
|
|
||||||
|
it_behaves_like 'pagination params'
|
||||||
|
end
|
||||||
|
|
||||||
|
def stub_request(path)
|
||||||
|
WebMock.stub_request(:get, "https://gitlab.com#{path}")
|
||||||
|
.to_return(status: 200)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -20,7 +20,7 @@ describe API::Environments do
|
||||||
path path_with_namespace
|
path path_with_namespace
|
||||||
star_count forks_count
|
star_count forks_count
|
||||||
created_at last_activity_at
|
created_at last_activity_at
|
||||||
avatar_url
|
avatar_url namespace
|
||||||
)
|
)
|
||||||
|
|
||||||
get api("/projects/#{project.id}/environments", user)
|
get api("/projects/#{project.id}/environments", user)
|
||||||
|
|
|
@ -225,7 +225,7 @@ describe API::Projects do
|
||||||
path path_with_namespace
|
path path_with_namespace
|
||||||
star_count forks_count
|
star_count forks_count
|
||||||
created_at last_activity_at
|
created_at last_activity_at
|
||||||
avatar_url
|
avatar_url namespace
|
||||||
)
|
)
|
||||||
|
|
||||||
get api('/projects?simple=true', user)
|
get api('/projects?simple=true', user)
|
||||||
|
|
Loading…
Reference in New Issue