Add latest changes from gitlab-org/gitlab@master

Gemfile.lock

@@ -9,7 +9,6 @@ GEM
   remote: https://rubygems.org/
   specs:
     RedCloth (4.3.2)
-    abstract_type (0.0.7)
     acme-client (2.0.6)
       faraday (>= 0.17, < 2.0.0)
     actioncable (6.1.3.2)
@@ -76,9 +75,6 @@ GEM
       zeitwerk (~> 2.3)
     acts-as-taggable-on (7.0.0)
       activerecord (>= 5.0, < 6.2)
-    adamantium (0.2.0)
-      ice_nine (~> 0.11.0)
-      memoizable (~> 0.4.0)
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aes_key_wrap (1.1.0)
@@ -205,9 +201,6 @@ GEM
     colored2 (3.1.2)
     commonmarker (0.21.0)
       ruby-enum (~> 0.5)
-    concord (0.1.5)
-      adamantium (~> 0.2.0)
-      equalizer (~> 0.0.9)
     concurrent-ruby (1.1.9)
     connection_pool (2.2.2)
     contracts (0.11.0)
@@ -336,7 +329,6 @@ GEM
       launchy (~> 2.1)
       mail (~> 2.7)
     encryptor (3.0.0)
-    equalizer (0.0.11)
     erubi (1.9.0)
     escape_utils (1.2.1)
     et-orbi (1.2.1)
@@ -647,7 +639,6 @@ GEM
       concurrent-ruby (~> 1.0)
     i18n_data (0.8.0)
     icalendar (2.4.1)
-    ice_nine (0.11.2)
     invisible_captcha (1.1.0)
       rails (>= 4.2)
     ipaddress (0.8.3)
@@ -748,8 +739,6 @@ GEM
       actionpack (>= 2.3)
       activerecord (>= 2.3)
     memoist (0.16.2)
-    memoizable (0.4.2)
-      thread_safe (~> 0.3, >= 0.3.1)
     memory_profiler (0.9.14)
     method_source (1.0.0)
     mime-types (3.3.1)
@@ -935,7 +924,6 @@ GEM
       coderay
       parser
       unparser
-    procto (0.0.3)
     prometheus-client-mmap (0.12.0)
     pry (0.13.1)
       coderay (~> 1.1)
@@ -1086,7 +1074,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-parameterized (0.4.2)
+    rspec-parameterized (0.5.0)
       binding_ninja (>= 0.2.3)
       parser
       proc_to_ast
@@ -1279,7 +1267,6 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thor (1.1.0)
-    thread_safe (0.3.6)
     thrift (0.14.0)
     tilt (2.0.10)
     timecop (0.9.1)
@@ -1332,14 +1319,9 @@ GEM
     uniform_notifier (1.13.0)
     unleash (0.1.5)
       murmurhash3 (~> 0.1.6)
-    unparser (0.4.7)
+    unparser (0.6.0)
-      abstract_type (~> 0.0.7)
-      adamantium (~> 0.2.0)
-      concord (~> 0.1.5)
       diff-lcs (~> 1.3)
-      equalizer (~> 0.0.9)
+      parser (>= 3.0.0)
-      parser (>= 2.6.5)
-      procto (~> 0.0.2)
     uri_template (0.7.0)
     valid_email (0.1.3)
       activemodel

app/workers/all_queues.yml

@@ -903,7 +903,7 @@
   :feature_category: :importers
   :has_external_dependencies: true
   :urgency: :low
-  :resource_boundary: :unknown
+  :resource_boundary: :cpu
   :weight: 1
   :idempotent:
   :tags:
@@ -913,7 +913,7 @@
   :feature_category: :importers
   :has_external_dependencies: true
   :urgency: :low
-  :resource_boundary: :unknown
+  :resource_boundary: :cpu
   :weight: 1
   :idempotent:
   :tags:

app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb

@@ -6,6 +6,7 @@ module Gitlab
       include ObjectImporter
 
       tags :exclude_from_kubernetes
+      worker_resource_boundary :cpu
 
       def representation_class
         Gitlab::GithubImport::Representation::PullRequest

app/workers/gitlab/github_import/import_pull_request_review_worker.rb

@@ -6,6 +6,7 @@ module Gitlab
       include ObjectImporter
 
       tags :exclude_from_kubernetes
+      worker_resource_boundary :cpu
 
       def representation_class
         Gitlab::GithubImport::Representation::PullRequestReview

doc/development/testing_guide/best_practices.md

@@ -972,11 +972,16 @@ range of inputs, might look like this:
 describe "#==" do
   using RSpec::Parameterized::TableSyntax
 
+  let(:one) { 1 }
+  let(:two) { 2 }
+
   where(:a, :b, :result) do
     1         | 1         | true
     1         | 2         | false
     true      | true      | true
     true      | false     | false
+    ref(:one) | ref(:one) | true  # let variables must be referenced using `ref`
+    ref(:one) | ref(:two) | false
   end
 
   with_them do

doc/user/compliance/license_compliance/index.md

@@ -83,6 +83,9 @@ The reported licenses might be incomplete or inaccurate.
 
 ## Requirements
 
+WARNING:
+License Compliance Scanning does not support run-time installation of compilers and interpreters.
+
 To run a License Compliance scanning job, you need GitLab Runner with the
 [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html).
 

doc/user/project/code_owners.md

@@ -8,23 +8,22 @@ type: reference
 # Code Owners **(PREMIUM)**
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6916) in GitLab 11.3.
-> - Code Owners for Merge Request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4418) in GitLab Premium 11.9.
+> - Code Owners for merge request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4418) in GitLab Premium 11.9.
 > - Moved to GitLab Premium in 13.9.
 
-Code Owners define who owns specific files or paths in a repository.
+Code Owners define who owns specific files or folders in a repository.
-You can require that Code Owners approve a merge request before it's merged.
 
-Code Owners help you determine who should review or approve merge requests.
+- The users you define as Code Owners are displayed in the UI when you browse directories.
-If you have a question about a file or feature, Code Owners
+- You can set your merge requests so they must be approved by Code Owners before merge.
-can help you find someone who knows the answer.
+- You can protect a branch and allow only Code Owners to approve changes to the branch.
 
 If you don't want to use Code Owners for approvals, you can
 [configure rules](merge_requests/approvals/rules.md) instead.
 
 ## Set up Code Owners
 
-You can specify users or [shared groups](members/share_project_with_groups.md)
+You can use Code Owners to specify users or [shared groups](members/share_project_with_groups.md)
-that are responsible for specific files and directories in a repository.
+that are responsible for specific files and folders in a repository.
 
 To set up Code Owners:
 
@@ -38,22 +37,28 @@ To set up Code Owners:
 1. In the file, enter text that follows one of these patterns:
 
    ```plaintext
-   # A member as Code Owner of a file
+   # Code Owners for a file
-   filename @username
+   filename @username1 @username2
 
-   # A member as Code Owner of a directory
+   # Code Owners for a directory
-   directory @username
+   foldername @username1 @username2
 
-   # All group members as Code Owners of a file
+   # All group members as Code Owners for a file
    filename @groupname
 
-   # All group members as Code Owners of a directory
+   # All group members as Code Owners for a folder
-   directory @groupname
+   foldername @groupname
    ```
 
-The Code Owners are displayed in the UI by the files or directory they apply to.
+The Code Owners are now displayed in the UI.
-These owners apply to this branch only. When you add new files to the repository,
+
-you should update the `CODEOWNERS` file.
+Next steps:
+
+- [Add Code Owners as merge request approvers](merge_requests/approvals/rules.md#code-owners-as-eligible-approvers).
+- Set up [Code Owner approval on a protected branch](protected_branches.md#require-code-owner-approval-on-a-protected-branch).
+
+NOTE:
+The Code Owners apply to the current branch only.
 
 ## When a file matches multiple `CODEOWNERS` entries
 
@@ -71,42 +76,6 @@ README.md @user1
 
 The user that would show for `README.md` would be `@user2`.
 
-## Approvals by Code Owners
-
-After you've added Code Owners to a project, you can configure it to
-be used for merge request approvals:
-
-- As [merge request eligible approvers](merge_requests/approvals/rules.md#code-owners-as-eligible-approvers).
-- As required approvers for [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch). **(PREMIUM)**
-
-Developer or higher [permissions](../permissions.md) are required to
-approve a merge request.
-
-After it's set, Code Owners are displayed in merge request widgets:
-
-![MR widget - Code Owners](img/code_owners_mr_widget_v12_4.png)
-
-While you can use the `CODEOWNERS` file in addition to Merge Request
-[Approval Rules](merge_requests/approvals/rules.md),
-you can also use it as the sole driver of merge request approvals
-without using [Approval Rules](merge_requests/approvals/rules.md):
-
-1. Create the file in one of the three locations specified above.
-1. Set the code owners as required approvers for
-   [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch).
-1. Use [the syntax of Code Owners files](code_owners.md)
-   to specify the actual owners and granular permissions.
-
-Using Code Owners in conjunction with [protected branches](protected_branches.md#require-code-owner-approval-on-a-protected-branch)
-prevents any user who is not specified in the `CODEOWNERS` file from pushing
-changes for the specified files/paths, except those included in the
-**Allowed to push** column. This allows for a more inclusive push strategy, as
-administrators don't have to restrict developers from pushing directly to the
-protected branch, but can restrict pushing to certain files where a review by
-Code Owners is required.
-
-[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5, users and groups who are allowed to push to protected branches do not require a merge request to merge their feature branches. Thus, they can skip merge request approval rules, Code Owners included.
-
 ## Groups as Code Owners
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) in GitLab 12.1.
@@ -154,7 +123,7 @@ file.md @group-x/subgroup-y
 file.md @group-x @group-x/subgroup-y
 ```
 
-### Code Owners Sections **(PREMIUM)**
+### Code Owners sections **(PREMIUM)**
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in GitLab Premium 13.2 behind a feature flag, enabled by default.
 > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4.
@@ -213,18 +182,18 @@ this example, entries defined under the sections "Documentation" and
 "DOCUMENTATION" would be combined into one, using the case of the first instance
 of the section encountered in the file.
 
-When assigned to a section, each code owner rule displayed in merge requests
+When assigned to a section, each Code Owner rule displayed in merge requests
 widget is sorted under a "section" label. In the screenshot below, we can see
 the rules for "Groups" and "Documentation" sections:
 
 ![MR widget - Sectional Code Owners](img/sectional_code_owners_v13.2.png)
 
-#### Optional Code Owners Sections **(PREMIUM)**
+#### Optional Code Owners sections **(PREMIUM)**
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232995) in GitLab Premium 13.8 behind a feature flag, enabled by default.
 > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53227) in GitLab 13.9.
 
-To make a certain section optional, add a code owners section prepended with the
+To make a certain section optional, add a Code Owners section prepended with the
 caret `^` character. Approvals from owners listed in the section are **not** required. For example:
 
 ```plaintext
@@ -238,13 +207,13 @@ caret `^` character. Approvals from owners listed in the section are **not** req
 *.go @root
 ```
 
-The optional code owners section displays in merge requests under the **Approval Rules** area:
+The optional Code Owners section displays in merge requests under the **Approval Rules** area:
 
-![MR widget - Optional Code Owners Sections](img/optional_code_owners_sections_v13_8.png)
+![MR widget - Optional Code Owners sections](img/optional_code_owners_sections_v13_8.png)
 
 If a section is duplicated in the file, and one of them is marked as optional and the other isn't, the requirement prevails.
 
-For example, the code owners of the "Documentation" section below is still required to approve merge requests:
+For example, the Code Owners of the "Documentation" section below is still required to approve merge requests:
 
 ```plaintext
 [Documentation]
@@ -260,9 +229,9 @@ For example, the code owners of the "Documentation" section below is still requi
 *.txt @root
 ```
 
-Optional sections in the code owners file are treated as optional only
+Optional sections in the `CODEOWNERS` file are treated as optional only
 when changes are submitted by using merge requests. If a change is submitted directly
-to the protected branch, approval from code owners is still required, even if the
+to the protected branch, approval from Code Owners is still required, even if the
 section is marked as optional. We plan to change this behavior in a
 [future release](https://gitlab.com/gitlab-org/gitlab/-/issues/297638),
 and allow direct pushes to the protected branch for sections marked as optional.
@@ -270,7 +239,7 @@ and allow direct pushes to the protected branch for sections marked as optional.
 ## Example `CODEOWNERS` file
 
 ```plaintext
-# This is an example of a code owners file
+# This is an example of a CODEOWNERS file
 # lines starting with a `#` will be ignored.
 
 # app/ @commented-rule
@@ -291,7 +260,7 @@ and allow direct pushes to the protected branch for sections marked as optional.
 
 # Multiple codeowners can be specified, separated by spaces or tabs
 # In the following case the CODEOWNERS file from the root of the repo
-# has 3 code owners (@multiple @code @owners)
+# has 3 Code Owners (@multiple @code @owners)
 CODEOWNERS @multiple @code @owners
 
 # Both usernames or email addresses can be used to match
@@ -304,11 +273,11 @@ LICENSE @legal this_does_not_match janedoe@gitlab.com
 # them as owners for a file
 README @group @group/with-nested/subgroup
 
-# Ending a path in a `/` will specify the code owners for every file
+# Ending a path in a `/` will specify the Code Owners for every file
 # nested in that directory, on any level
 /docs/ @all-docs
 
-# Ending a path in `/*` will specify code owners for every file in
+# Ending a path in `/*` will specify Code Owners for every file in
 # that directory, but not nested deeper. This will match
 # `docs/index.md` but not `docs/projects/index.md`
 /docs/* @root-docs
@@ -321,7 +290,7 @@ lib/ @lib-owner
 # repository
 /config/ @config-owner
 
-# If the path contains spaces, these need to be escaped like this:
+# If the path contains spaces, escape them like this:
 path\ with\ spaces/ @space-owner
 
 # Code Owners section:

doc/user/project/protected_branches.md

@@ -179,8 +179,7 @@ When enabled, members who are can push to this branch can also force push.
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13251) in GitLab Premium 12.4.
 > - [In](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.5 and later, users and groups who can push to protected branches do not have to use a merge request to merge their feature branches. This means they can skip merge request approval rules.
 
-You can require at least one approval by a [Code Owner](code_owners.md) to a file changed by the
+For a protected branch, you can require at least one approval by a [Code Owner](code_owners.md).
-merge request.
 
 To protect a new branch and enable Code Owner's approval:
 
@@ -201,6 +200,16 @@ When enabled, all merge requests for these branches require approval
 by a Code Owner per matched rule before they can be merged.
 Additionally, direct pushes to the protected branch are denied if a rule is matched.
 
+Any user who is not specified in the `CODEOWNERS` file cannot push
+changes for the specified files or paths, unless they are specifically allowed to.
+You don't have to restrict developers from pushing directly to the
+protected branch. Instead, you can restrict pushing to certain files where a review by
+Code Owners is required.
+
+In [GitLab Premium 13.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/35097), users and groups
+who are allowed to push to protected branches do not need a merge request to merge their feature branches.
+Thus, they can skip merge request approval rules, Code Owners included.
+
 ## Run pipelines on protected branches
 
 The permission to merge or push to protected branches defines

qa/qa/resource/issue.rb

@@ -14,11 +14,11 @@ module QA
         end
       end
 
-      attribute :id
+      attributes :id,
-      attribute :iid
+                 :iid,
-      attribute :assignee_ids
+                 :assignee_ids,
-      attribute :labels
+                 :labels,
-      attribute :title
+                 :title
 
       def initialize
         @assignee_ids = []
@@ -41,13 +41,21 @@ module QA
       end
 
       def api_get_path
-        "/projects/#{project.id}/issues/#{id}"
+        "/projects/#{project.id}/issues/#{iid}"
       end
 
       def api_post_path
         "/projects/#{project.id}/issues"
       end
 
+      def api_put_path
+        "/projects/#{project.id}/issues/#{iid}"
+      end
+
+      def api_comments_path
+        "#{api_get_path}/notes"
+      end
+
       def api_post_body
         {
           assignee_ids: assignee_ids,
@@ -59,20 +67,28 @@ module QA
         end
       end
 
-      def api_put_path
-        "/projects/#{project.id}/issues/#{iid}"
-      end
-
       def set_issue_assignees(assignee_ids:)
         put_body = { assignee_ids: assignee_ids }
         response = put Runtime::API::Request.new(api_client, api_put_path).url, put_body
 
         unless response.code == HTTP_STATUS_OK
-          raise ResourceUpdateFailedError, "Could not update issue assignees to #{assignee_ids}. Request returned (#{response.code}): `#{response}`."
+          raise(
+            ResourceUpdateFailedError,
+            "Could not update issue assignees to #{assignee_ids}. Request returned (#{response.code}): `#{response}`."
+          )
         end
 
         QA::Runtime::Logger.debug("Successfully updated issue assignees to #{assignee_ids}")
       end
+
+      # Get issue comments
+      #
+      # @return [Array]
+      def comments(auto_paginate: false)
+        return parse_body(api_get_from(api_comments_path)) unless auto_paginate
+
+        auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url)
+      end
     end
   end
 end

qa/qa/resource/merge_request.rb

@@ -160,9 +160,10 @@ module QA
       # Get MR comments
       #
       # @return [Array]
-      def comments
+      def comments(auto_paginate: false)
-        response = get(Runtime::API::Request.new(api_client, api_comments_path).url)
+        return parse_body(api_get_from(api_comments_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url)
       end
 
       private

qa/qa/resource/project.rb

@@ -32,7 +32,7 @@ module QA
       end
 
       attribute :path_with_namespace do
-        "#{sandbox_path}#{group.path}/#{name}" if group
+        "#{group.full_path}/#{name}"
       end
 
       alias_method :full_path, :path_with_namespace
@@ -268,14 +268,16 @@ module QA
         result[:import_status]
       end
 
-      def commits
+      def commits(auto_paginate: false)
-        response = get(request_url(api_commits_path))
+        return parse_body(api_get_from(api_commits_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_commits_path, per_page: '100'))
       end
 
-      def merge_requests
+      def merge_requests(auto_paginate: false)
-        response = get(request_url(api_merge_requests_path))
+        return parse_body(api_get_from(api_merge_requests_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_merge_requests_path, per_page: '100'))
       end
 
       def merge_request_with_title(title)
@@ -299,9 +301,10 @@ module QA
         parse_body(response)
       end
 
-      def repository_branches
+      def repository_branches(auto_paginate: false)
-        response = get(request_url(api_repository_branches_path))
+        return parse_body(api_get_from(api_repository_branches_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_repository_branches_path, per_page: '100'))
       end
 
       def repository_tags
@@ -324,19 +327,22 @@ module QA
         parse_body(response)
       end
 
-      def issues
+      def issues(auto_paginate: false)
-        response = get(request_url(api_issues_path))
+        return parse_body(api_get_from(api_issues_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_issues_path, per_page: '100'))
       end
 
-      def labels
+      def labels(auto_paginate: false)
-        response = get(request_url(api_labels_path))
+        return parse_body(api_get_from(api_labels_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_labels_path, per_page: '100'))
       end
 
-      def milestones
+      def milestones(auto_paginate: false)
-        response = get(request_url(api_milestones_path))
+        return parse_body(api_get_from(api_milestones_path)) unless auto_paginate
-        parse_body(response)
+
+        auto_paginated_response(request_url(api_milestones_path, per_page: '100'))
       end
 
       def wikis

qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require 'octokit'
+require 'parallel'
+
+# rubocop:disable Rails/Pluck
+module QA
+  # Only executes in custom job/pipeline
+  RSpec.describe 'Manage', :github, :requires_admin, only: { job: 'large-github-import' } do
+    describe 'Project import' do
+      let(:api_client) { Runtime::API::Client.as_admin }
+      let(:group) do
+        Resource::Group.fabricate_via_api! do |resource|
+          resource.api_client = api_client
+        end
+      end
+
+      let(:user) do
+        Resource::User.fabricate_via_api! do |resource|
+          resource.api_client = api_client
+          resource.hard_delete_on_api_removal = true
+        end
+      end
+
+      let(:differ) { RSpec::Support::Differ.new(color: true) }
+      let(:github_repo) { 'allure-framework/allure-ruby' }
+
+      let(:github_client) do
+        Octokit.middleware = Faraday::RackBuilder.new do |builder|
+          builder.response(:logger, Runtime::Logger.logger, headers: false, bodies: false)
+        end
+
+        Octokit::Client.new(access_token: Runtime::Env.github_access_token, auto_paginate: true)
+      end
+
+      let(:gh_branches) { github_client.branches(github_repo).map(&:name) }
+      let(:gh_commits) { github_client.commits(github_repo).map(&:sha) }
+      let(:gh_repo) { github_client.repository(github_repo) }
+      let(:gh_labels) { github_client.labels(github_repo) }
+      let(:gh_milestones) { github_client.list_milestones(github_repo, state: 'all') }
+
+      let(:gh_all_issues) do
+        github_client.list_issues(github_repo, state: 'all')
+      end
+
+      let(:gh_prs) do
+        gh_all_issues.select(&:pull_request).each_with_object({}) do |pr, hash|
+          hash[pr.title] = {
+            body: pr.body || '',
+            comments: [*gh_pr_comments[pr.html_url], *gh_issue_comments[pr.html_url]].compact.sort
+          }
+        end
+      end
+
+      let(:gh_issues) do
+        gh_all_issues.reject(&:pull_request).each_with_object({}) do |issue, hash|
+          hash[issue.title] = {
+            body: issue.body || '',
+            comments: gh_issue_comments[issue.html_url]
+          }
+        end
+      end
+
+      let(:gh_issue_comments) do
+        github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
+          hash[c.html_url.gsub(/\#\S+/, "")] << c.body # use base html url as key
+        end
+      end
+
+      let(:gh_pr_comments) do
+        github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
+          hash[c.html_url.gsub(/\#\S+/, "")] << c.body # use base html url as key
+        end
+      end
+
+      let(:imported_project) do
+        Resource::ProjectImportedFromGithub.fabricate_via_api! do |project|
+          project.add_name_uuid = false
+          project.name = 'imported-project'
+          project.group = group
+          project.github_personal_access_token = Runtime::Env.github_access_token
+          project.github_repository_path = github_repo
+          project.api_client = api_client
+        end
+      end
+
+      before do
+        group.add_member(user, Resource::Members::AccessLevel::MAINTAINER)
+      end
+
+      it 'imports large Github repo via api' do
+        imported_project # import the project
+        fetch_github_objects # fetch all objects right after import has started
+
+        expect { imported_project.reload!.import_status }.to eventually_eq('finished').within(
+          duration: 3600,
+          interval: 30
+        )
+
+        aggregate_failures do
+          verify_repository_import
+          verify_merge_requests_import
+          verify_issues_import
+          verify_labels_import
+          verify_milestones_import
+        end
+      end
+
+      # Persist all objects from repository being imported
+      #
+      # @return [void]
+      def fetch_github_objects
+        Runtime::Logger.debug("Fetching objects for github repo: '#{github_repo}'")
+
+        gh_repo
+        gh_branches
+        gh_commits
+        gh_prs
+        gh_issues
+        gh_labels
+        gh_milestones
+      end
+
+      # Verify repository imported correctly
+      #
+      # @return [void]
+      def verify_repository_import
+        branches = imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] }
+        commits = imported_project.commits(auto_paginate: true).map { |c| c[:id] }
+
+        expect(imported_project.description).to eq(gh_repo.description)
+        # check via include, importer creates more branches
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/332711
+        expect(branches).to include(*gh_branches)
+        expect(commits).to match_array(gh_commits)
+      end
+
+      # Verify imported merge requests and mr issues
+      #
+      # @return [void]
+      def verify_merge_requests_import
+        verify_mrs_or_issues('mrs')
+      end
+
+      # Verify imported issues and issue comments
+      #
+      # @return [void]
+      def verify_issues_import
+        verify_mrs_or_issues('issues')
+      end
+
+      # Verify imported labels
+      #
+      # @return [void]
+      def verify_labels_import
+        labels = imported_project.labels(auto_paginate: true).map { |label| label.slice(:name, :color) }
+        actual_labels = gh_labels.map { |label| { name: label.name, color: "##{label.color}" } }
+
+        expect(labels.length).to eq(actual_labels.length)
+        expect(labels).to match_array(actual_labels)
+      end
+
+      # Verify milestones import
+      #
+      # @return [void]
+      def verify_milestones_import
+        milestones = imported_project.milestones(auto_paginate: true).map { |ms| ms.slice(:title, :description) }
+        actual_milestones = gh_milestones.map { |ms| { title: ms.title, description: ms.description } }
+
+        expect(milestones.length).to eq(actual_milestones.length)
+        expect(milestones).to match_array(actual_milestones)
+      end
+
+      private
+
+      # Verify imported mrs or issues
+      #
+      # @param [String] type verification object, 'mrs' or 'issues'
+      # @return [void]
+      def verify_mrs_or_issues(type)
+        msg = ->(title) { "expected #{type} with title '#{title}' to have" }
+        expected = type == 'mrs' ? mrs : gl_issues
+        actual = type == 'mrs' ? gh_prs : gh_issues
+
+        expect(expected.keys).to match_array(actual.keys)
+        actual.each do |title, actual_item|
+          expected_item = expected[title]
+
+          expect(expected_item).to be_truthy, "#{msg.call(title)} been imported"
+          next unless expected_item
+
+          expect(expected_item[:body]).to(
+            include(actual_item[:body]),
+            "#{msg.call(title)} same description. #{diff(expected_item[:body], actual_item[:body])}"
+          )
+          expect(expected_item[:comments].length).to(
+            eq(actual_item[:comments].length),
+            "#{msg.call(title)} same amount of comments"
+          )
+          expect(expected_item[:comments]).to match_array(actual_item[:comments])
+        end
+      end
+
+      # Imported project merge requests
+      #
+      # @return [Hash]
+      def mrs
+        @mrs ||= begin
+          imported_mrs = imported_project.merge_requests(auto_paginate: true)
+          # fetch comments in parallel since we need to do it for each mr separately
+          mrs_hashes = Parallel.map(imported_mrs, in_processes: 5) do |mr|
+            resource = Resource::MergeRequest.init do |resource|
+              resource.project = imported_project
+              resource.iid = mr[:iid]
+              resource.api_client = api_client
+            end
+
+            {
+              title: mr[:title],
+              body: mr[:description],
+              comments: resource.comments(auto_paginate: true)
+                # remove system notes
+                .reject { |c| c[:system] || c[:body].match?(/^(\*\*Review:\*\*)|(\*Merged by:).*/) }
+                .map { |c| sanitize(c[:body]) }
+            }
+          end
+
+          mrs_hashes.each_with_object({}) do |mr, hash|
+            hash[mr[:title]] = {
+              body: mr[:body],
+              comments: mr[:comments]
+            }
+          end
+        end
+      end
+
+      # Imported project issues
+      #
+      # @return [Hash]
+      def gl_issues
+        @gl_issues ||= begin
+          imported_issues = imported_project.issues(auto_paginate: true)
+          # fetch comments in parallel since we need to do it for each mr separately
+          issue_hashes = Parallel.map(imported_issues, in_processes: 5) do |issue|
+            resource = Resource::Issue.init do |issue_resource|
+              issue_resource.project = imported_project
+              issue_resource.iid = issue[:iid]
+              issue_resource.api_client = api_client
+            end
+
+            {
+              title: issue[:title],
+              body: issue[:description],
+              comments: resource.comments(auto_paginate: true).map { |c| sanitize(c[:body]) }
+            }
+          end
+
+          issue_hashes.each_with_object({}) do |issue, hash|
+            hash[issue[:title]] = {
+              body: issue[:body],
+              comments: issue[:comments]
+            }
+          end
+        end
+      end
+
+      # Remove added prefixes by importer
+      #
+      # @param [String] body
+      # @return [String]
+      def sanitize(body)
+        body.gsub(/\*Created by: \S+\*\n\n/, "")
+      end
+
+      # Diff of 2 objects
+      #
+      # @param [Object] actual
+      # @param [Object] expected
+      # @return [String]
+      def diff(actual, expected)
+        "diff:\n#{differ.diff(actual, expected)}"
+      end
+    end
+  end
+end
+# rubocop:enable Rails/Pluck

qa/qa/support/api.rb

@@ -79,11 +79,20 @@ module QA
         error.response
       end
 
+      def auto_paginated_response(url)
+        pages = []
+        with_paginated_response_body(url) { |response| pages << response }
+
+        pages.flatten
+      end
+
       def with_paginated_response_body(url)
         loop do
           response = get(url)
+          page, pages = response.headers.values_at(:x_page, :x_total_pages)
+          api_endpoint = url.match(%r{v4/(\S+)\?})[1]
 
-          QA::Runtime::Logger.debug("Fetching page #{response.headers[:x_page]} of #{response.headers[:x_total_pages]}...")
+          QA::Runtime::Logger.debug("Fetching page (#{page}/#{pages}) for '#{api_endpoint}' ...") unless pages.to_i <= 1
 
           yield parse_body(response)
 
@@ -96,7 +105,7 @@ module QA
 
       def pagination_links(response)
         response.headers[:link].split(',').map do |link|
-          match = link.match(/\<(?<url>.*)\>\; rel=\"(?<rel>\w+)\"/)
+          match = link.match(/<(?<url>.*)>; rel="(?<rel>\w+)"/)
           break nil unless match
 
           { url: match[:url], rel: match[:rel] }

qa/spec/support/matchers/eventually_matcher.rb

@@ -24,6 +24,7 @@ module Matchers
       chain(:within) do |options = {}|
         @duration = options[:duration]
         @attempts = options[:attempts]
+        @interval = options[:interval]
       end
 
       def supports_block_expectations?
@@ -55,7 +56,7 @@ module Matchers
         QA::Support::Retrier.retry_until(
           max_attempts: @attempts,
           max_duration: @duration,
-          sleep_interval: 0.5
+          sleep_interval: @interval || 0.5
         ) do
           public_send(expectation_name, actual)
         rescue RSpec::Expectations::ExpectationNotMetError, QA::Resource::ApiFabricator::ResourceNotFoundError

spec/features/projects/blobs/blob_show_spec.rb

@@ -13,6 +13,22 @@ RSpec.describe 'File blob', :js do
     wait_for_requests
   end
 
+  def create_file(file_name, content)
+    project.add_maintainer(project.creator)
+
+    Files::CreateService.new(
+      project,
+      project.creator,
+      start_branch: 'master',
+      branch_name: 'master',
+      commit_message: "Add #{file_name}",
+      file_path: file_name,
+      file_content: <<-SPEC.strip_heredoc
+        #{content}
+      SPEC
+    ).execute
+  end
+
   context 'Ruby file' do
     before do
       visit_blob('files/ruby/popen.rb')
@@ -785,6 +801,255 @@ RSpec.describe 'File blob', :js do
         end
       end
     end
+
+    context 'CONTRIBUTING.md' do
+      before do
+        file_name = 'CONTRIBUTING.md'
+
+        create_file(file_name, '## Contribution guidelines')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("After you've reviewed these contribution guidelines, you'll be all set to contribute to this project.")
+        end
+      end
+    end
+
+    context 'CHANGELOG.md' do
+      before do
+        file_name = 'CHANGELOG.md'
+
+        create_file(file_name, '## Changelog for v1.0.0')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("To find the state of this project's repository at the time of any of these versions, check out the tags.")
+        end
+      end
+    end
+
+    context 'Cargo.toml' do
+      before do
+        file_name = 'Cargo.toml'
+
+        create_file(file_name, '
+            [package]
+            name = "hello_world" # the name of the package
+            version = "0.1.0"    # the current version, obeying semver
+            authors = ["Alice <a@example.com>", "Bob <b@example.com>"]
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Cargo.")
+        end
+      end
+    end
+
+    context 'Cartfile' do
+      before do
+        file_name = 'Cartfile'
+
+        create_file(file_name, '
+            gitlab "Alamofire/Alamofire" == 4.9.0
+            gitlab "Alamofire/AlamofireImage" ~> 3.4
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Carthage.")
+        end
+      end
+    end
+
+    context 'composer.json' do
+      before do
+        file_name = 'composer.json'
+
+        create_file(file_name, '
+            {
+              "license": "MIT"
+            }
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Composer.")
+        end
+      end
+    end
+
+    context 'Gemfile' do
+      before do
+        file_name = 'Gemfile'
+
+        create_file(file_name, '
+            source "https://rubygems.org"
+
+            # Gems here
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Bundler.")
+        end
+      end
+    end
+
+    context 'Godeps.json' do
+      before do
+        file_name = 'Godeps.json'
+
+        create_file(file_name, '
+            {
+              "GoVersion": "go1.6"
+            }
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using godep.")
+        end
+      end
+    end
+
+    context 'go.mod' do
+      before do
+        file_name = 'go.mod'
+
+        create_file(file_name, '
+            module example.com/mymodule
+
+            go 1.14
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Go Modules.")
+        end
+      end
+    end
+
+    context 'package.json' do
+      before do
+        file_name = 'package.json'
+
+        create_file(file_name, '
+            {
+              "name": "my-awesome-package",
+              "version": "1.0.0"
+            }
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using npm.")
+        end
+      end
+    end
+
+    context 'podfile' do
+      before do
+        file_name = 'podfile'
+
+        create_file(file_name, 'platform :ios, "8.0"')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using CocoaPods.")
+        end
+      end
+    end
+
+    context 'test.podspec' do
+      before do
+        file_name = 'test.podspec'
+
+        create_file(file_name, '
+            Pod::Spec.new do |s|
+              s.name = "TensorFlowLiteC"
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using CocoaPods.")
+        end
+      end
+    end
+
+    context 'JSON.podspec.json' do
+      before do
+        file_name = 'JSON.podspec.json'
+
+        create_file(file_name, '
+            {
+              "name": "JSON"
+            }
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using CocoaPods.")
+        end
+      end
+    end
+
+    context 'requirements.txt' do
+      before do
+        file_name = 'requirements.txt'
+
+        create_file(file_name, 'Project requirements')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using pip.")
+        end
+      end
+    end
+
+    context 'yarn.lock' do
+      before do
+        file_name = 'yarn.lock'
+
+        create_file(file_name, '
+            # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+            # yarn lockfile v1
+          ')
+        visit_blob(file_name)
+      end
+
+      it 'displays an auxiliary viewer' do
+        aggregate_failures do
+          expect(page).to have_content("This project manages its dependencies using Yarn.")
+        end
+      end
+    end
   end
 
   context 'realtime pipelines' do