diff --git a/Gemfile.lock b/Gemfile.lock index 652f60f93b8..617e1f18caa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,6 @@ GEM remote: https://rubygems.org/ specs: RedCloth (4.3.2) - abstract_type (0.0.7) acme-client (2.0.6) faraday (>= 0.17, < 2.0.0) actioncable (6.1.3.2) @@ -76,9 +75,6 @@ GEM zeitwerk (~> 2.3) acts-as-taggable-on (7.0.0) activerecord (>= 5.0, < 6.2) - adamantium (0.2.0) - ice_nine (~> 0.11.0) - memoizable (~> 0.4.0) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) @@ -205,9 +201,6 @@ GEM colored2 (3.1.2) commonmarker (0.21.0) ruby-enum (~> 0.5) - concord (0.1.5) - adamantium (~> 0.2.0) - equalizer (~> 0.0.9) concurrent-ruby (1.1.9) connection_pool (2.2.2) contracts (0.11.0) @@ -336,7 +329,6 @@ GEM launchy (~> 2.1) mail (~> 2.7) encryptor (3.0.0) - equalizer (0.0.11) erubi (1.9.0) escape_utils (1.2.1) et-orbi (1.2.1) @@ -647,7 +639,6 @@ GEM concurrent-ruby (~> 1.0) i18n_data (0.8.0) icalendar (2.4.1) - ice_nine (0.11.2) invisible_captcha (1.1.0) rails (>= 4.2) ipaddress (0.8.3) @@ -748,8 +739,6 @@ GEM actionpack (>= 2.3) activerecord (>= 2.3) memoist (0.16.2) - memoizable (0.4.2) - thread_safe (~> 0.3, >= 0.3.1) memory_profiler (0.9.14) method_source (1.0.0) mime-types (3.3.1) @@ -935,7 +924,6 @@ GEM coderay parser unparser - procto (0.0.3) prometheus-client-mmap (0.12.0) pry (0.13.1) coderay (~> 1.1) @@ -1086,7 +1074,7 @@ GEM rspec-mocks (3.10.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.10.0) - rspec-parameterized (0.4.2) + rspec-parameterized (0.5.0) binding_ninja (>= 0.2.3) parser proc_to_ast @@ -1279,7 +1267,6 @@ GEM eventmachine (~> 1.0, >= 1.0.4) rack (>= 1, < 3) thor (1.1.0) - thread_safe (0.3.6) thrift (0.14.0) tilt (2.0.10) timecop (0.9.1) @@ -1332,14 +1319,9 @@ GEM uniform_notifier (1.13.0) unleash (0.1.5) murmurhash3 (~> 0.1.6) - unparser (0.4.7) - abstract_type (~> 0.0.7) - adamantium (~> 0.2.0) - concord (~> 0.1.5) + unparser (0.6.0) diff-lcs (~> 1.3) - equalizer (~> 0.0.9) - parser (>= 2.6.5) - procto (~> 0.0.2) + parser (>= 3.0.0) uri_template (0.7.0) valid_email (0.1.3) activemodel diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 8d08beb56aa..e002039d226 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -903,7 +903,7 @@ :feature_category: :importers :has_external_dependencies: true :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: @@ -913,7 +913,7 @@ :feature_category: :importers :has_external_dependencies: true :urgency: :low - :resource_boundary: :unknown + :resource_boundary: :cpu :weight: 1 :idempotent: :tags: diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index 91dab3470d9..cce179542c7 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequest diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index de10fe40589..8796d6392df 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequestReview diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index e153fa9f334..6b6fbe3996e 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -972,11 +972,16 @@ range of inputs, might look like this: describe "#==" do using RSpec::Parameterized::TableSyntax + let(:one) { 1 } + let(:two) { 2 } + where(:a, :b, :result) do - 1 | 1 | true - 1 | 2 | false - true | true | true - true | false | false + 1 | 1 | true + 1 | 2 | false + true | true | true + true | false | false + ref(:one) | ref(:one) | true # let variables must be referenced using `ref` + ref(:one) | ref(:two) | false end with_them do diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 1a43c5ae96f..e39a3f7111b 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -83,6 +83,9 @@ The reported licenses might be incomplete or inaccurate. ## Requirements +WARNING: +License Compliance Scanning does not support run-time installation of compilers and interpreters. + To run a License Compliance scanning job, you need GitLab Runner with the [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html). diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 2a60c06814b..32ad0db1866 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -8,23 +8,22 @@ type: reference # Code Owners **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6916) in GitLab 11.3. -> - Code Owners for Merge Request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4418) in GitLab Premium 11.9. +> - Code Owners for merge request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4418) in GitLab Premium 11.9. > - Moved to GitLab Premium in 13.9. -Code Owners define who owns specific files or paths in a repository. -You can require that Code Owners approve a merge request before it's merged. +Code Owners define who owns specific files or folders in a repository. -Code Owners help you determine who should review or approve merge requests. -If you have a question about a file or feature, Code Owners -can help you find someone who knows the answer. +- The users you define as Code Owners are displayed in the UI when you browse directories. +- You can set your merge requests so they must be approved by Code Owners before merge. +- You can protect a branch and allow only Code Owners to approve changes to the branch. If you don't want to use Code Owners for approvals, you can [configure rules](merge_requests/approvals/rules.md) instead. ## Set up Code Owners -You can specify users or [shared groups](members/share_project_with_groups.md) -that are responsible for specific files and directories in a repository. +You can use Code Owners to specify users or [shared groups](members/share_project_with_groups.md) +that are responsible for specific files and folders in a repository. To set up Code Owners: @@ -38,22 +37,28 @@ To set up Code Owners: 1. In the file, enter text that follows one of these patterns: ```plaintext - # A member as Code Owner of a file - filename @username + # Code Owners for a file + filename @username1 @username2 - # A member as Code Owner of a directory - directory @username + # Code Owners for a directory + foldername @username1 @username2 - # All group members as Code Owners of a file + # All group members as Code Owners for a file filename @groupname - # All group members as Code Owners of a directory - directory @groupname + # All group members as Code Owners for a folder + foldername @groupname ``` -The Code Owners are displayed in the UI by the files or directory they apply to. -These owners apply to this branch only. When you add new files to the repository, -you should update the `CODEOWNERS` file. +The Code Owners are now displayed in the UI. + +Next steps: + +- [Add Code Owners as merge request approvers](merge_requests/approvals/rules.md#code-owners-as-eligible-approvers). +- Set up [Code Owner approval on a protected branch](protected_branches.md#require-code-owner-approval-on-a-protected-branch). + +NOTE: +The Code Owners apply to the current branch only. ## When a file matches multiple `CODEOWNERS` entries @@ -71,42 +76,6 @@ README.md @user1 The user that would show for `README.md` would be `@user2`. -## Approvals by Code Owners - -After you've added Code Owners to a project, you can configure it to -be used for merge request approvals: - -- As [merge request eligible approvers](merge_requests/approvals/rules.md#code-owners-as-eligible-approvers). -- As required approvers for [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch). **(PREMIUM)** - -Developer or higher [permissions](../permissions.md) are required to -approve a merge request. - -After it's set, Code Owners are displayed in merge request widgets: - -![MR widget - Code Owners](img/code_owners_mr_widget_v12_4.png) - -While you can use the `CODEOWNERS` file in addition to Merge Request -[Approval Rules](merge_requests/approvals/rules.md), -you can also use it as the sole driver of merge request approvals -without using [Approval Rules](merge_requests/approvals/rules.md): - -1. Create the file in one of the three locations specified above. -1. Set the code owners as required approvers for - [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch). -1. Use [the syntax of Code Owners files](code_owners.md) - to specify the actual owners and granular permissions. - -Using Code Owners in conjunction with [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch) -prevents any user who is not specified in the `CODEOWNERS` file from pushing -changes for the specified files/paths, except those included in the -**Allowed to push** column. This allows for a more inclusive push strategy, as -administrators don't have to restrict developers from pushing directly to the -protected branch, but can restrict pushing to certain files where a review by -Code Owners is required. - -[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5, users and groups who are allowed to push to protected branches do not require a merge request to merge their feature branches. Thus, they can skip merge request approval rules, Code Owners included. - ## Groups as Code Owners > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) in GitLab 12.1. @@ -154,7 +123,7 @@ file.md @group-x/subgroup-y file.md @group-x @group-x/subgroup-y ``` -### Code Owners Sections **(PREMIUM)** +### Code Owners sections **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in GitLab Premium 13.2 behind a feature flag, enabled by default. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4. @@ -213,18 +182,18 @@ this example, entries defined under the sections "Documentation" and "DOCUMENTATION" would be combined into one, using the case of the first instance of the section encountered in the file. -When assigned to a section, each code owner rule displayed in merge requests +When assigned to a section, each Code Owner rule displayed in merge requests widget is sorted under a "section" label. In the screenshot below, we can see the rules for "Groups" and "Documentation" sections: ![MR widget - Sectional Code Owners](img/sectional_code_owners_v13.2.png) -#### Optional Code Owners Sections **(PREMIUM)** +#### Optional Code Owners sections **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232995) in GitLab Premium 13.8 behind a feature flag, enabled by default. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53227) in GitLab 13.9. -To make a certain section optional, add a code owners section prepended with the +To make a certain section optional, add a Code Owners section prepended with the caret `^` character. Approvals from owners listed in the section are **not** required. For example: ```plaintext @@ -238,13 +207,13 @@ caret `^` character. Approvals from owners listed in the section are **not** req *.go @root ``` -The optional code owners section displays in merge requests under the **Approval Rules** area: +The optional Code Owners section displays in merge requests under the **Approval Rules** area: -![MR widget - Optional Code Owners Sections](img/optional_code_owners_sections_v13_8.png) +![MR widget - Optional Code Owners sections](img/optional_code_owners_sections_v13_8.png) If a section is duplicated in the file, and one of them is marked as optional and the other isn't, the requirement prevails. -For example, the code owners of the "Documentation" section below is still required to approve merge requests: +For example, the Code Owners of the "Documentation" section below is still required to approve merge requests: ```plaintext [Documentation] @@ -260,9 +229,9 @@ For example, the code owners of the "Documentation" section below is still requi *.txt @root ``` -Optional sections in the code owners file are treated as optional only +Optional sections in the `CODEOWNERS` file are treated as optional only when changes are submitted by using merge requests. If a change is submitted directly -to the protected branch, approval from code owners is still required, even if the +to the protected branch, approval from Code Owners is still required, even if the section is marked as optional. We plan to change this behavior in a [future release](https://gitlab.com/gitlab-org/gitlab/-/issues/297638), and allow direct pushes to the protected branch for sections marked as optional. @@ -270,7 +239,7 @@ and allow direct pushes to the protected branch for sections marked as optional. ## Example `CODEOWNERS` file ```plaintext -# This is an example of a code owners file +# This is an example of a CODEOWNERS file # lines starting with a `#` will be ignored. # app/ @commented-rule @@ -291,7 +260,7 @@ and allow direct pushes to the protected branch for sections marked as optional. # Multiple codeowners can be specified, separated by spaces or tabs # In the following case the CODEOWNERS file from the root of the repo -# has 3 code owners (@multiple @code @owners) +# has 3 Code Owners (@multiple @code @owners) CODEOWNERS @multiple @code @owners # Both usernames or email addresses can be used to match @@ -304,11 +273,11 @@ LICENSE @legal this_does_not_match janedoe@gitlab.com # them as owners for a file README @group @group/with-nested/subgroup -# Ending a path in a `/` will specify the code owners for every file +# Ending a path in a `/` will specify the Code Owners for every file # nested in that directory, on any level /docs/ @all-docs -# Ending a path in `/*` will specify code owners for every file in +# Ending a path in `/*` will specify Code Owners for every file in # that directory, but not nested deeper. This will match # `docs/index.md` but not `docs/projects/index.md` /docs/* @root-docs @@ -321,7 +290,7 @@ lib/ @lib-owner # repository /config/ @config-owner -# If the path contains spaces, these need to be escaped like this: +# If the path contains spaces, escape them like this: path\ with\ spaces/ @space-owner # Code Owners section: diff --git a/doc/user/project/img/code_owners_mr_widget_v12_4.png b/doc/user/project/img/code_owners_mr_widget_v12_4.png deleted file mode 100644 index 7f7b15ee017..00000000000 Binary files a/doc/user/project/img/code_owners_mr_widget_v12_4.png and /dev/null differ diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index 4ff651891b2..2e39be57a0e 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -179,8 +179,7 @@ When enabled, members who are can push to this branch can also force push. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13251) in GitLab Premium 12.4. > - [In](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5 and later, users and groups who can push to protected branches do not have to use a merge request to merge their feature branches. This means they can skip merge request approval rules. -You can require at least one approval by a [Code Owner](code_owners.md) to a file changed by the -merge request. +For a protected branch, you can require at least one approval by a [Code Owner](code_owners.md). To protect a new branch and enable Code Owner's approval: @@ -201,6 +200,16 @@ When enabled, all merge requests for these branches require approval by a Code Owner per matched rule before they can be merged. Additionally, direct pushes to the protected branch are denied if a rule is matched. +Any user who is not specified in the `CODEOWNERS` file cannot push +changes for the specified files or paths, unless they are specifically allowed to. +You don't have to restrict developers from pushing directly to the +protected branch. Instead, you can restrict pushing to certain files where a review by +Code Owners is required. + +In [GitLab Premium 13.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/35097), users and groups +who are allowed to push to protected branches do not need a merge request to merge their feature branches. +Thus, they can skip merge request approval rules, Code Owners included. + ## Run pipelines on protected branches The permission to merge or push to protected branches defines diff --git a/qa/qa/resource/issue.rb b/qa/qa/resource/issue.rb index ffffa0eecda..2144e619c70 100644 --- a/qa/qa/resource/issue.rb +++ b/qa/qa/resource/issue.rb @@ -14,11 +14,11 @@ module QA end end - attribute :id - attribute :iid - attribute :assignee_ids - attribute :labels - attribute :title + attributes :id, + :iid, + :assignee_ids, + :labels, + :title def initialize @assignee_ids = [] @@ -41,13 +41,21 @@ module QA end def api_get_path - "/projects/#{project.id}/issues/#{id}" + "/projects/#{project.id}/issues/#{iid}" end def api_post_path "/projects/#{project.id}/issues" end + def api_put_path + "/projects/#{project.id}/issues/#{iid}" + end + + def api_comments_path + "#{api_get_path}/notes" + end + def api_post_body { assignee_ids: assignee_ids, @@ -59,20 +67,28 @@ module QA end end - def api_put_path - "/projects/#{project.id}/issues/#{iid}" - end - def set_issue_assignees(assignee_ids:) put_body = { assignee_ids: assignee_ids } response = put Runtime::API::Request.new(api_client, api_put_path).url, put_body unless response.code == HTTP_STATUS_OK - raise ResourceUpdateFailedError, "Could not update issue assignees to #{assignee_ids}. Request returned (#{response.code}): `#{response}`." + raise( + ResourceUpdateFailedError, + "Could not update issue assignees to #{assignee_ids}. Request returned (#{response.code}): `#{response}`." + ) end QA::Runtime::Logger.debug("Successfully updated issue assignees to #{assignee_ids}") end + + # Get issue comments + # + # @return [Array] + def comments(auto_paginate: false) + return parse_body(api_get_from(api_comments_path)) unless auto_paginate + + auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url) + end end end end diff --git a/qa/qa/resource/merge_request.rb b/qa/qa/resource/merge_request.rb index 8d9de0ea718..419893f0b11 100644 --- a/qa/qa/resource/merge_request.rb +++ b/qa/qa/resource/merge_request.rb @@ -160,9 +160,10 @@ module QA # Get MR comments # # @return [Array] - def comments - response = get(Runtime::API::Request.new(api_client, api_comments_path).url) - parse_body(response) + def comments(auto_paginate: false) + return parse_body(api_get_from(api_comments_path)) unless auto_paginate + + auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url) end private diff --git a/qa/qa/resource/project.rb b/qa/qa/resource/project.rb index 70cec0cf747..12ae644cab8 100644 --- a/qa/qa/resource/project.rb +++ b/qa/qa/resource/project.rb @@ -32,7 +32,7 @@ module QA end attribute :path_with_namespace do - "#{sandbox_path}#{group.path}/#{name}" if group + "#{group.full_path}/#{name}" end alias_method :full_path, :path_with_namespace @@ -268,14 +268,16 @@ module QA result[:import_status] end - def commits - response = get(request_url(api_commits_path)) - parse_body(response) + def commits(auto_paginate: false) + return parse_body(api_get_from(api_commits_path)) unless auto_paginate + + auto_paginated_response(request_url(api_commits_path, per_page: '100')) end - def merge_requests - response = get(request_url(api_merge_requests_path)) - parse_body(response) + def merge_requests(auto_paginate: false) + return parse_body(api_get_from(api_merge_requests_path)) unless auto_paginate + + auto_paginated_response(request_url(api_merge_requests_path, per_page: '100')) end def merge_request_with_title(title) @@ -299,9 +301,10 @@ module QA parse_body(response) end - def repository_branches - response = get(request_url(api_repository_branches_path)) - parse_body(response) + def repository_branches(auto_paginate: false) + return parse_body(api_get_from(api_repository_branches_path)) unless auto_paginate + + auto_paginated_response(request_url(api_repository_branches_path, per_page: '100')) end def repository_tags @@ -324,19 +327,22 @@ module QA parse_body(response) end - def issues - response = get(request_url(api_issues_path)) - parse_body(response) + def issues(auto_paginate: false) + return parse_body(api_get_from(api_issues_path)) unless auto_paginate + + auto_paginated_response(request_url(api_issues_path, per_page: '100')) end - def labels - response = get(request_url(api_labels_path)) - parse_body(response) + def labels(auto_paginate: false) + return parse_body(api_get_from(api_labels_path)) unless auto_paginate + + auto_paginated_response(request_url(api_labels_path, per_page: '100')) end - def milestones - response = get(request_url(api_milestones_path)) - parse_body(response) + def milestones(auto_paginate: false) + return parse_body(api_get_from(api_milestones_path)) unless auto_paginate + + auto_paginated_response(request_url(api_milestones_path, per_page: '100')) end def wikis diff --git a/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb b/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb new file mode 100644 index 00000000000..22917d0d146 --- /dev/null +++ b/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb @@ -0,0 +1,286 @@ +# frozen_string_literal: true + +require 'octokit' +require 'parallel' + +# rubocop:disable Rails/Pluck +module QA + # Only executes in custom job/pipeline + RSpec.describe 'Manage', :github, :requires_admin, only: { job: 'large-github-import' } do + describe 'Project import' do + let(:api_client) { Runtime::API::Client.as_admin } + let(:group) do + Resource::Group.fabricate_via_api! do |resource| + resource.api_client = api_client + end + end + + let(:user) do + Resource::User.fabricate_via_api! do |resource| + resource.api_client = api_client + resource.hard_delete_on_api_removal = true + end + end + + let(:differ) { RSpec::Support::Differ.new(color: true) } + let(:github_repo) { 'allure-framework/allure-ruby' } + + let(:github_client) do + Octokit.middleware = Faraday::RackBuilder.new do |builder| + builder.response(:logger, Runtime::Logger.logger, headers: false, bodies: false) + end + + Octokit::Client.new(access_token: Runtime::Env.github_access_token, auto_paginate: true) + end + + let(:gh_branches) { github_client.branches(github_repo).map(&:name) } + let(:gh_commits) { github_client.commits(github_repo).map(&:sha) } + let(:gh_repo) { github_client.repository(github_repo) } + let(:gh_labels) { github_client.labels(github_repo) } + let(:gh_milestones) { github_client.list_milestones(github_repo, state: 'all') } + + let(:gh_all_issues) do + github_client.list_issues(github_repo, state: 'all') + end + + let(:gh_prs) do + gh_all_issues.select(&:pull_request).each_with_object({}) do |pr, hash| + hash[pr.title] = { + body: pr.body || '', + comments: [*gh_pr_comments[pr.html_url], *gh_issue_comments[pr.html_url]].compact.sort + } + end + end + + let(:gh_issues) do + gh_all_issues.reject(&:pull_request).each_with_object({}) do |issue, hash| + hash[issue.title] = { + body: issue.body || '', + comments: gh_issue_comments[issue.html_url] + } + end + end + + let(:gh_issue_comments) do + github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| + hash[c.html_url.gsub(/\#\S+/, "")] << c.body # use base html url as key + end + end + + let(:gh_pr_comments) do + github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| + hash[c.html_url.gsub(/\#\S+/, "")] << c.body # use base html url as key + end + end + + let(:imported_project) do + Resource::ProjectImportedFromGithub.fabricate_via_api! do |project| + project.add_name_uuid = false + project.name = 'imported-project' + project.group = group + project.github_personal_access_token = Runtime::Env.github_access_token + project.github_repository_path = github_repo + project.api_client = api_client + end + end + + before do + group.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + end + + it 'imports large Github repo via api' do + imported_project # import the project + fetch_github_objects # fetch all objects right after import has started + + expect { imported_project.reload!.import_status }.to eventually_eq('finished').within( + duration: 3600, + interval: 30 + ) + + aggregate_failures do + verify_repository_import + verify_merge_requests_import + verify_issues_import + verify_labels_import + verify_milestones_import + end + end + + # Persist all objects from repository being imported + # + # @return [void] + def fetch_github_objects + Runtime::Logger.debug("Fetching objects for github repo: '#{github_repo}'") + + gh_repo + gh_branches + gh_commits + gh_prs + gh_issues + gh_labels + gh_milestones + end + + # Verify repository imported correctly + # + # @return [void] + def verify_repository_import + branches = imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] } + commits = imported_project.commits(auto_paginate: true).map { |c| c[:id] } + + expect(imported_project.description).to eq(gh_repo.description) + # check via include, importer creates more branches + # https://gitlab.com/gitlab-org/gitlab/-/issues/332711 + expect(branches).to include(*gh_branches) + expect(commits).to match_array(gh_commits) + end + + # Verify imported merge requests and mr issues + # + # @return [void] + def verify_merge_requests_import + verify_mrs_or_issues('mrs') + end + + # Verify imported issues and issue comments + # + # @return [void] + def verify_issues_import + verify_mrs_or_issues('issues') + end + + # Verify imported labels + # + # @return [void] + def verify_labels_import + labels = imported_project.labels(auto_paginate: true).map { |label| label.slice(:name, :color) } + actual_labels = gh_labels.map { |label| { name: label.name, color: "##{label.color}" } } + + expect(labels.length).to eq(actual_labels.length) + expect(labels).to match_array(actual_labels) + end + + # Verify milestones import + # + # @return [void] + def verify_milestones_import + milestones = imported_project.milestones(auto_paginate: true).map { |ms| ms.slice(:title, :description) } + actual_milestones = gh_milestones.map { |ms| { title: ms.title, description: ms.description } } + + expect(milestones.length).to eq(actual_milestones.length) + expect(milestones).to match_array(actual_milestones) + end + + private + + # Verify imported mrs or issues + # + # @param [String] type verification object, 'mrs' or 'issues' + # @return [void] + def verify_mrs_or_issues(type) + msg = ->(title) { "expected #{type} with title '#{title}' to have" } + expected = type == 'mrs' ? mrs : gl_issues + actual = type == 'mrs' ? gh_prs : gh_issues + + expect(expected.keys).to match_array(actual.keys) + actual.each do |title, actual_item| + expected_item = expected[title] + + expect(expected_item).to be_truthy, "#{msg.call(title)} been imported" + next unless expected_item + + expect(expected_item[:body]).to( + include(actual_item[:body]), + "#{msg.call(title)} same description. #{diff(expected_item[:body], actual_item[:body])}" + ) + expect(expected_item[:comments].length).to( + eq(actual_item[:comments].length), + "#{msg.call(title)} same amount of comments" + ) + expect(expected_item[:comments]).to match_array(actual_item[:comments]) + end + end + + # Imported project merge requests + # + # @return [Hash] + def mrs + @mrs ||= begin + imported_mrs = imported_project.merge_requests(auto_paginate: true) + # fetch comments in parallel since we need to do it for each mr separately + mrs_hashes = Parallel.map(imported_mrs, in_processes: 5) do |mr| + resource = Resource::MergeRequest.init do |resource| + resource.project = imported_project + resource.iid = mr[:iid] + resource.api_client = api_client + end + + { + title: mr[:title], + body: mr[:description], + comments: resource.comments(auto_paginate: true) + # remove system notes + .reject { |c| c[:system] || c[:body].match?(/^(\*\*Review:\*\*)|(\*Merged by:).*/) } + .map { |c| sanitize(c[:body]) } + } + end + + mrs_hashes.each_with_object({}) do |mr, hash| + hash[mr[:title]] = { + body: mr[:body], + comments: mr[:comments] + } + end + end + end + + # Imported project issues + # + # @return [Hash] + def gl_issues + @gl_issues ||= begin + imported_issues = imported_project.issues(auto_paginate: true) + # fetch comments in parallel since we need to do it for each mr separately + issue_hashes = Parallel.map(imported_issues, in_processes: 5) do |issue| + resource = Resource::Issue.init do |issue_resource| + issue_resource.project = imported_project + issue_resource.iid = issue[:iid] + issue_resource.api_client = api_client + end + + { + title: issue[:title], + body: issue[:description], + comments: resource.comments(auto_paginate: true).map { |c| sanitize(c[:body]) } + } + end + + issue_hashes.each_with_object({}) do |issue, hash| + hash[issue[:title]] = { + body: issue[:body], + comments: issue[:comments] + } + end + end + end + + # Remove added prefixes by importer + # + # @param [String] body + # @return [String] + def sanitize(body) + body.gsub(/\*Created by: \S+\*\n\n/, "") + end + + # Diff of 2 objects + # + # @param [Object] actual + # @param [Object] expected + # @return [String] + def diff(actual, expected) + "diff:\n#{differ.diff(actual, expected)}" + end + end + end +end +# rubocop:enable Rails/Pluck diff --git a/qa/qa/support/api.rb b/qa/qa/support/api.rb index de9da3171b0..1493feeeed7 100644 --- a/qa/qa/support/api.rb +++ b/qa/qa/support/api.rb @@ -79,11 +79,20 @@ module QA error.response end + def auto_paginated_response(url) + pages = [] + with_paginated_response_body(url) { |response| pages << response } + + pages.flatten + end + def with_paginated_response_body(url) loop do response = get(url) + page, pages = response.headers.values_at(:x_page, :x_total_pages) + api_endpoint = url.match(%r{v4/(\S+)\?})[1] - QA::Runtime::Logger.debug("Fetching page #{response.headers[:x_page]} of #{response.headers[:x_total_pages]}...") + QA::Runtime::Logger.debug("Fetching page (#{page}/#{pages}) for '#{api_endpoint}' ...") unless pages.to_i <= 1 yield parse_body(response) @@ -96,7 +105,7 @@ module QA def pagination_links(response) response.headers[:link].split(',').map do |link| - match = link.match(/\<(?.*)\>\; rel=\"(?\w+)\"/) + match = link.match(/<(?.*)>; rel="(?\w+)"/) break nil unless match { url: match[:url], rel: match[:rel] } diff --git a/qa/spec/support/matchers/eventually_matcher.rb b/qa/spec/support/matchers/eventually_matcher.rb index 3f0afd6fb54..2de6598be4c 100644 --- a/qa/spec/support/matchers/eventually_matcher.rb +++ b/qa/spec/support/matchers/eventually_matcher.rb @@ -24,6 +24,7 @@ module Matchers chain(:within) do |options = {}| @duration = options[:duration] @attempts = options[:attempts] + @interval = options[:interval] end def supports_block_expectations? @@ -55,7 +56,7 @@ module Matchers QA::Support::Retrier.retry_until( max_attempts: @attempts, max_duration: @duration, - sleep_interval: 0.5 + sleep_interval: @interval || 0.5 ) do public_send(expectation_name, actual) rescue RSpec::Expectations::ExpectationNotMetError, QA::Resource::ApiFabricator::ResourceNotFoundError diff --git a/spec/features/projects/blobs/blob_show_spec.rb b/spec/features/projects/blobs/blob_show_spec.rb index 677052eebea..9e4cec200d4 100644 --- a/spec/features/projects/blobs/blob_show_spec.rb +++ b/spec/features/projects/blobs/blob_show_spec.rb @@ -13,6 +13,22 @@ RSpec.describe 'File blob', :js do wait_for_requests end + def create_file(file_name, content) + project.add_maintainer(project.creator) + + Files::CreateService.new( + project, + project.creator, + start_branch: 'master', + branch_name: 'master', + commit_message: "Add #{file_name}", + file_path: file_name, + file_content: <<-SPEC.strip_heredoc + #{content} + SPEC + ).execute + end + context 'Ruby file' do before do visit_blob('files/ruby/popen.rb') @@ -785,6 +801,255 @@ RSpec.describe 'File blob', :js do end end end + + context 'CONTRIBUTING.md' do + before do + file_name = 'CONTRIBUTING.md' + + create_file(file_name, '## Contribution guidelines') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("After you've reviewed these contribution guidelines, you'll be all set to contribute to this project.") + end + end + end + + context 'CHANGELOG.md' do + before do + file_name = 'CHANGELOG.md' + + create_file(file_name, '## Changelog for v1.0.0') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("To find the state of this project's repository at the time of any of these versions, check out the tags.") + end + end + end + + context 'Cargo.toml' do + before do + file_name = 'Cargo.toml' + + create_file(file_name, ' + [package] + name = "hello_world" # the name of the package + version = "0.1.0" # the current version, obeying semver + authors = ["Alice ", "Bob "] + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Cargo.") + end + end + end + + context 'Cartfile' do + before do + file_name = 'Cartfile' + + create_file(file_name, ' + gitlab "Alamofire/Alamofire" == 4.9.0 + gitlab "Alamofire/AlamofireImage" ~> 3.4 + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Carthage.") + end + end + end + + context 'composer.json' do + before do + file_name = 'composer.json' + + create_file(file_name, ' + { + "license": "MIT" + } + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Composer.") + end + end + end + + context 'Gemfile' do + before do + file_name = 'Gemfile' + + create_file(file_name, ' + source "https://rubygems.org" + + # Gems here + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Bundler.") + end + end + end + + context 'Godeps.json' do + before do + file_name = 'Godeps.json' + + create_file(file_name, ' + { + "GoVersion": "go1.6" + } + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using godep.") + end + end + end + + context 'go.mod' do + before do + file_name = 'go.mod' + + create_file(file_name, ' + module example.com/mymodule + + go 1.14 + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Go Modules.") + end + end + end + + context 'package.json' do + before do + file_name = 'package.json' + + create_file(file_name, ' + { + "name": "my-awesome-package", + "version": "1.0.0" + } + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using npm.") + end + end + end + + context 'podfile' do + before do + file_name = 'podfile' + + create_file(file_name, 'platform :ios, "8.0"') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using CocoaPods.") + end + end + end + + context 'test.podspec' do + before do + file_name = 'test.podspec' + + create_file(file_name, ' + Pod::Spec.new do |s| + s.name = "TensorFlowLiteC" + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using CocoaPods.") + end + end + end + + context 'JSON.podspec.json' do + before do + file_name = 'JSON.podspec.json' + + create_file(file_name, ' + { + "name": "JSON" + } + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using CocoaPods.") + end + end + end + + context 'requirements.txt' do + before do + file_name = 'requirements.txt' + + create_file(file_name, 'Project requirements') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using pip.") + end + end + end + + context 'yarn.lock' do + before do + file_name = 'yarn.lock' + + create_file(file_name, ' + # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. + # yarn lockfile v1 + ') + visit_blob(file_name) + end + + it 'displays an auxiliary viewer' do + aggregate_failures do + expect(page).to have_content("This project manages its dependencies using Yarn.") + end + end + end end context 'realtime pipelines' do