diff --git a/app/controllers/import/gitlab_controller.rb b/app/controllers/import/gitlab_controller.rb index fccbdbca0f6..53f70446d95 100644 --- a/app/controllers/import/gitlab_controller.rb +++ b/app/controllers/import/gitlab_controller.rb @@ -1,4 +1,7 @@ class Import::GitlabController < Import::BaseController + MAX_PROJECT_PAGES = 15 + PER_PAGE_PROJECTS = 100 + before_action :verify_gitlab_import_enabled before_action :gitlab_auth, except: :callback @@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController end def status - @repos = client.projects + @repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS) @already_added_projects = find_already_added_projects('gitlab') already_added_projects_names = @already_added_projects.pluck(:import_source) diff --git a/changelogs/unreleased/fj-48123-fix-gitlab-import.yml b/changelogs/unreleased/fj-48123-fix-gitlab-import.yml new file mode 100644 index 00000000000..896db2cdcb8 --- /dev/null +++ b/changelogs/unreleased/fj-48123-fix-gitlab-import.yml @@ -0,0 +1,5 @@ +--- +title: Fix GitLab project imports not loading due to API timeouts +merge_request: 20599 +author: +type: fixed diff --git a/doc/api/projects.md b/doc/api/projects.md index f3ccca46420..9409afc88a8 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the "avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png", "star_count":3812, "forks_count":3561, - "last_activity_at":"2018-01-02T11:40:26.570Z" + "last_activity_at":"2018-01-02T11:40:26.570Z", + "namespace": { + "id": 72, + "name": "GitLab.org", + "path": "gitlab-org", + "kind": "group", + "full_path": "gitlab-org", + "parent_id": null + } } ... diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8c40d66c651..464a31ee819 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -132,6 +132,7 @@ module API expose :star_count, :forks_count expose :last_activity_at + expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes def self.preload_relation(projects_relation, options = {}) @@ -194,7 +195,6 @@ module API expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace, using: 'API::Entities::NamespaceBasic' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 22719e9a003..38ef12491df 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -32,15 +32,15 @@ module Gitlab api.get("/api/v4/user").parsed end - def issues(project_identifier) - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + def issues(project_identifier, **kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed end end - def issue_comments(project_identifier, issue_id) - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + def issue_comments(project_identifier, issue_id, **kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed end end @@ -48,23 +48,27 @@ module Gitlab api.get("/api/v4/projects/#{id}").parsed end - def projects - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + def projects(**kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed end end private - def lazy_page_iterator(per_page) + def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE) Enumerator.new do |y| - page = 1 + page = starting_page + page_limit = (starting_page - 1) + page_limit if page_limit + loop do - items = yield(page) + items = yield(page, per_page) + items.each do |item| y << item end - break if items.empty? || items.size < per_page + + break if items.empty? || items.size < per_page || (page_limit && page >= page_limit) page += 1 end diff --git a/spec/fixtures/api/schemas/public_api/v4/projects.json b/spec/fixtures/api/schemas/public_api/v4/projects.json index 17ad8d8c48d..af5670ebd33 100644 --- a/spec/fixtures/api/schemas/public_api/v4/projects.json +++ b/spec/fixtures/api/schemas/public_api/v4/projects.json @@ -24,13 +24,24 @@ "avatar_url": { "type": ["string", "null"] }, "star_count": { "type": "integer" }, "forks_count": { "type": "integer" }, - "last_activity_at": { "type": "date" } + "last_activity_at": { "type": "date" }, + "namespace": { + "type": "object", + "properties" : { + "id": { "type": "integer" }, + "name": { "type": "string" }, + "path": { "type": "string" }, + "kind": { "type": "string" }, + "full_path": { "type": "string" }, + "parent_id": { "type": ["integer", "null"] } + } + } }, "required": [ "id", "name", "name_with_namespace", "description", "path", "path_with_namespace", "created_at", "default_branch", "tag_list", "ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url", - "star_count", "last_activity_at" + "star_count", "last_activity_at", "namespace" ], "additionalProperties": false } diff --git a/spec/lib/gitlab/gitlab_import/client_spec.rb b/spec/lib/gitlab/gitlab_import/client_spec.rb index 50e8d7183ce..22ad88e28cb 100644 --- a/spec/lib/gitlab/gitlab_import/client_spec.rb +++ b/spec/lib/gitlab/gitlab_import/client_spec.rb @@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do expect(key).to be_kind_of(Symbol) end end + + it 'uses membership and simple flags' do + stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true') + + expect_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([]) + + expect(client.projects.to_a).to eq [] + end + + shared_examples 'pagination params' do + before do + allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([]) + end + + it 'allows page_limit param' do + allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return(element_list) + + expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original + + client.send(method, *args, page_limit: 2, per_page: 1).to_a + end + + it 'allows per_page param' do + expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original + + client.send(method, *args, per_page: 2).to_a + end + + it 'allows starting_page param' do + expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original + + client.send(method, *args, starting_page: 3).to_a + end + end + + describe '#projects' do + subject(:method) { :projects } + let(:args) { [] } + let(:element_list) { build_list(:project, 2) } + + before do + stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true') + stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true') + stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true') + stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true') + end + + it_behaves_like 'pagination params' + end + + describe '#issues' do + subject(:method) { :issues } + let(:args) { [1] } + let(:element_list) { build_list(:issue, 2) } + + before do + stub_request('/api/v4/projects/1/issues?page=1&per_page=1') + stub_request('/api/v4/projects/1/issues?page=2&per_page=1') + stub_request('/api/v4/projects/1/issues?page=1&per_page=2') + stub_request('/api/v4/projects/1/issues?page=3&per_page=100') + end + + it_behaves_like 'pagination params' + end + + describe '#issue_comments' do + subject(:method) { :issue_comments } + let(:args) { [1, 1] } + let(:element_list) { build_list(:note_on_issue, 2) } + + before do + stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1') + stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1') + stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2') + stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100') + end + + it_behaves_like 'pagination params' + end + + def stub_request(path) + WebMock.stub_request(:get, "https://gitlab.com#{path}") + .to_return(status: 200) + end end diff --git a/spec/requests/api/environments_spec.rb b/spec/requests/api/environments_spec.rb index bf93555b0f0..f3db0c122a0 100644 --- a/spec/requests/api/environments_spec.rb +++ b/spec/requests/api/environments_spec.rb @@ -20,7 +20,7 @@ describe API::Environments do path path_with_namespace star_count forks_count created_at last_activity_at - avatar_url + avatar_url namespace ) get api("/projects/#{project.id}/environments", user) diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb index f72c01561d8..5ac008c7e40 100644 --- a/spec/requests/api/projects_spec.rb +++ b/spec/requests/api/projects_spec.rb @@ -225,7 +225,7 @@ describe API::Projects do path path_with_namespace star_count forks_count created_at last_activity_at - avatar_url + avatar_url namespace ) get api('/projects?simple=true', user)