Add latest changes from gitlab-org/gitlab@master

.gitpod.yml

@@ -48,6 +48,8 @@ tasks:
           make gitlab-db-migrate
         fi
         cd /workspace/gitlab-development-kit/gitlab
+        # Display which branch we're on
+        git branch --show-current
         # Install Lefthook
         bundle exec lefthook install
         git checkout db/structure.sql

app/views/admin/application_settings/_help_page.html.haml

@@ -7,20 +7,22 @@
     .form-group
       = f.label :help_page_text, _('Additional text to show on the Help page'), class: 'label-bold'
       = f.text_area :help_page_text, class: 'form-control gl-form-input', rows: 4
-      .form-text.text-muted= _('Markdown enabled')
+      .form-text.text-muted= _('Markdown enabled.')
     .form-group
       .form-check
         = f.check_box :help_page_hide_commercial_content, class: 'form-check-input'
         = f.label :help_page_hide_commercial_content, class: 'form-check-label' do
-          = _('Hide marketing-related entries from the Help page.')
+          = _('Hide marketing-related entries from the Help page')
     .form-group
       = f.label :help_page_support_url, _('Support page URL'), class: 'label-bold'
-      = f.text_field :help_page_support_url, class: 'form-control gl-form-input', placeholder: 'http://company.example.com/getting-help', :'aria-describedby' => 'support_help_block'
+      = f.text_field :help_page_support_url, class: 'form-control gl-form-input', placeholder: 'https://company.example.com/getting-help', :'aria-describedby' => 'support_help_block'
-      %span.form-text.text-muted#support_help_block= _('Alternate support URL for Help page and Help dropdown')
+      %span.form-text.text-muted#support_help_block= _('Alternate support URL for Help page and Help dropdown.')
 
     - if show_documentation_base_url_field?
       .form-group
         = f.label :help_page_documentation_base_url, _('Documentation pages URL'), class: 'label-bold'
         = f.text_field :help_page_documentation_base_url, class: 'form-control gl-form-input', placeholder: 'https://docs.gitlab.com'
-
+        - docs_link_url = help_page_path('user/admin_area/settings/help_page', anchor: 'destination-requirements')
+        - docs_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_link_url }
+        %span.form-text.text-muted#support_help_block= html_escape(_('Requests for pages at %{code_start}%{help_text_url}%{code_end} redirect to the URL. The destination must meet certain requirements. %{docs_link_start}Learn more.%{docs_link_end}')) % { code_start: '<code>'.html_safe, help_text_url: help_url, code_end: '</code>'.html_safe, docs_link_start: docs_link_start, docs_link_end: '</a>'.html_safe }
   = f.submit _('Save changes'), class: "gl-button btn btn-confirm"

doc/api/index.md

@@ -166,6 +166,11 @@ curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/ap
 
 Read more about [GitLab as an OAuth2 provider](oauth2.md).
 
+NOTE:
+We recommend that OAuth access tokens have an expiration. You can use a `refresh_token` to refresh tokens. Integrations may need to be updated to refresh tokens prior to expiration, which is based on the [expires_in](https://datatracker.ietf.org/doc/html/rfc6749#appendix-A.14) property in the token endpoint response.  
+
+A default refresh setting of two hours is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/336598).
+
 ### Personal/project access tokens
 
 You can use access tokens to authenticate with the API by passing it in either

doc/ssh/index.md

@@ -214,9 +214,12 @@ To use SSH with GitLab, copy your public key to your GitLab account.
 1. In the **Title** text box, type a description, like _Work Laptop_ or
    _Home Workstation_.
 1. Optional. In the **Expires at** box, select an expiration date. (Introduced in [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/36243).)
-   The expiration date is informational only, and does not prevent you from using
+   In:
-   the key. However, administrators can view expiration dates and
+   - GitLab 13.12 and earlier, the expiration date is informational only. It doesn't prevent
-   use them for guidance when [deleting keys](../user/admin_area/credentials_inventory.md#delete-a-users-ssh-key).
+     you from using the key. Administrators can view expiration dates and use them for
+     guidance when [deleting keys](../user/admin_area/credentials_inventory.md#delete-a-users-ssh-key).
+   - GitLab 14.0 and later, the expiration date is enforced. Administrators can
+     [allow expired keys to be used](../user/admin_area/settings/account_and_limit_settings.md#allow-expired-ssh-keys-to-be-used).
    - GitLab checks all SSH keys at 02:00 AM UTC every day. It emails an expiration notice for all SSH keys that expire on the current date. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322637) in GitLab 13.11.)
    - GitLab checks all SSH keys at 01:00 AM UTC every day. It emails an expiration notice for all SSH keys that are scheduled to expire seven days from now. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322637) in GitLab 13.11.)
 1. Select **Add key**.

doc/user/admin_area/settings/help_page.md

@@ -66,18 +66,24 @@ You can specify a custom URL to which users are directed when they:
 1. In the **Support page URL** field, enter the URL.
 1. Select **Save changes**.
 
-## Redirect GitLab documentation links
+## Redirect `/help` pages
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43157) in GitLab 13.5.
-> - Enabled on GitLab.com and is ready for production use. Available to GitLab.com administrators only.
+> - Enabled on GitLab.com and is ready for production use.
 
 NOTE:
 On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to
 [enable the `:help_page_documentation_redirect` flag](../../../administration/feature_flags.md).
 On GitLab.com, this feature is available but can be configured by GitLab.com administrators only.
 
-Documentation links go to the `/help` section on the instance by default, but you can
+The `/help` URL of a GitLab instance displays a basic version of the documentation sourced from the
-redirect these links to an external documentation site like `https://docs.gitlab.com`:
+[`doc` directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) of GitLab. `/help` links
+are often used for contextual help.
+
+You can redirect these `/help` links to either:
+
+- The more navigable and searchable version published at [`docs.gitlab.com`](https://docs.gitlab.com).
+- A destination that meets [necessary requirements](#destination-requirements).
 
 1. On the top bar, select **Menu >** **{admin}** **Admin**.
 1. In the left sidebar, select **Settings > Preferences**.
@@ -85,6 +91,18 @@ redirect these links to an external documentation site like `https://docs.gitlab
 1. In the **Documentation pages URL** field, enter the URL.
 1. Select **Save changes**.
 
+### Destination requirements
+
+When redirecting `/help`, GitLab:
+
+- Redirects requests to the specified URL.
+- Appends `ee`  and the documentation path to the URL.
+- Appends `.html` to the URL, and removes `.md` if necessary.
+
+For example, if the URL is set to `https://docs.gitlab.com`, requests for
+`/help/user/admin_area/settings/help_page.md` redirect to:
+`https://docs.gitlab.com/ee/user/admin_area/settings/help_page.html`.
+
 <!-- ## Troubleshooting
 
 Include any troubleshooting steps that you can foresee. If you know beforehand what issues

doc/user/application_security/api_fuzzing/index.md

@@ -38,7 +38,7 @@ or other scanners) during a scan could cause inaccurate results.
 
 You can run a Web API fuzzing scan using the following methods:
 
-- [OpenAPI Specification](#openapi-specification) - version 2.0 or 3.0
+- [OpenAPI Specification](#openapi-specification) - version 2, and 3.
 - [HTTP Archive](#http-archive-har) (HAR)
 - [Postman Collection](#postman-collection) - version 2.0 or 2.1
 
@@ -112,6 +112,8 @@ To generate an API Fuzzing configuration snippet:
 
 ### OpenAPI Specification
 
+> Support for OpenAPI Specification v3.1 was
+> [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327268) in GitLab 14.2.
 > Support for OpenAPI Specification using YAML format was
 > [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330583) in GitLab 14.0.
 > Support for OpenAPI Specification v3.0 was

doc/user/project/settings/project_access_tokens.md

@@ -95,7 +95,7 @@ You may enable or disable project access token creation for all projects in a gr
 Even when creation is disabled, you can still use and revoke existing project access tokens.
 This setting is available only on top-level groups.
 
-## Group access token workaround  **(FREE SELF)**
+## Group access token workaround **(FREE SELF)**
 
 NOTE:
 This section describes a workaround and is subject to change.

lib/gitlab/checks/changes_access.rb

@@ -40,7 +40,7 @@ module Gitlab
 
         return [] if newrevs.empty?
 
-        @commits ||= project.repository.new_commits(newrevs)
+        @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true)
       end
 
       # All commits which have been newly introduced via the given revision.

lib/gitlab/git/repository.rb

@@ -354,9 +354,9 @@ module Gitlab
         end
       end
 
-      def new_commits(newrevs)
+      def new_commits(newrevs, allow_quarantine: false)
         wrapped_gitaly_errors do
-          gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all])
+          gitaly_commit_client.list_new_commits(Array.wrap(newrevs), allow_quarantine: allow_quarantine)
         end
       end
 

lib/gitlab/gitaly_client/commit_service.rb

@@ -265,6 +265,31 @@ module Gitlab
         consume_commits_response(response)
       end
 
+      # List all commits which are new in the repository. If commits have been pushed into the repo
+      def list_new_commits(revisions, allow_quarantine: false)
+        git_env = Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository)
+        if allow_quarantine && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present?
+          # If we have a quarantine environment, then we can optimize the check
+          # by doing a ListAllCommitsRequest. Instead of walking through
+          # references, we just walk through all quarantined objects, which is
+          # a lot more efficient. To do so, we throw away any alternate object
+          # directories, which point to the main object directory of the
+          # repository, and only keep the object directory which points into
+          # the quarantine object directory.
+          quarantined_repo = @gitaly_repo.dup
+          quarantined_repo.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string)
+
+          request = Gitaly::ListAllCommitsRequest.new(
+            repository: quarantined_repo
+          )
+
+          response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout)
+          consume_commits_response(response)
+        else
+          list_commits(Array.wrap(revisions) + %w[--not --all])
+        end
+      end
+
       def list_commits_by_oid(oids)
         return [] if oids.empty?
 

locale/gitlab.pot

@@ -3420,7 +3420,7 @@ msgstr ""
 msgid "Also unassign this user from related issues and merge requests"
 msgstr ""
 
-msgid "Alternate support URL for Help page and Help dropdown"
+msgid "Alternate support URL for Help page and Help dropdown."
 msgstr ""
 
 msgid "Alternatively, you can convert your account to a managed account by the %{group_name} group."
@@ -16363,7 +16363,7 @@ msgstr ""
 msgid "Hide list"
 msgstr ""
 
-msgid "Hide marketing-related entries from the Help page."
+msgid "Hide marketing-related entries from the Help page"
 msgstr ""
 
 msgid "Hide payload"
@@ -20211,7 +20211,7 @@ msgstr ""
 msgid "Markdown Help"
 msgstr ""
 
-msgid "Markdown enabled"
+msgid "Markdown enabled."
 msgstr ""
 
 msgid "Markdown is supported"
@@ -28080,6 +28080,9 @@ msgstr ""
 msgid "Requests Profiles"
 msgstr ""
 
+msgid "Requests for pages at %{code_start}%{help_text_url}%{code_end} redirect to the URL. The destination must meet certain requirements. %{docs_link_start}Learn more.%{docs_link_end}"
+msgstr ""
+
 msgid "Requests per period"
 msgstr ""
 

spec/features/admin/admin_settings_spec.rb

@@ -585,7 +585,7 @@ RSpec.describe 'Admin updates settings' do
 
         page.within('.as-help-page') do
           fill_in 'Additional text to show on the Help page', with: 'Example text'
-          check 'Hide marketing-related entries from the Help page.'
+          check 'Hide marketing-related entries from the Help page'
           fill_in 'Support page URL', with: new_support_url
           fill_in 'Documentation pages URL', with: new_documentation_url
           click_button 'Save changes'

spec/lib/gitlab/checks/changes_access_spec.rb

@@ -70,7 +70,7 @@ RSpec.describe Gitlab::Checks::ChangesAccess do
       let(:expected_commit) { instance_double(Commit) }
 
       it 'returns only commits with non empty revisions' do
-        expect(project.repository).to receive(:new_commits).with([newrev]) { [expected_commit] }
+        expect(project.repository).to receive(:new_commits).with([newrev], { allow_quarantine: true }) { [expected_commit] }
         expect(subject.commits).to eq([expected_commit])
       end
     end

spec/lib/gitlab/gitaly_client/commit_service_spec.rb

@@ -343,6 +343,92 @@ RSpec.describe Gitlab::GitalyClient::CommitService do
     end
   end
 
+  describe '#list_new_commits' do
+    let(:revisions) { [revision] }
+    let(:gitaly_commits) { create_list(:gitaly_commit, 3) }
+    let(:commits) { gitaly_commits.map { |c| Gitlab::Git::Commit.new(repository, c) }}
+
+    subject { client.list_new_commits(revisions, allow_quarantine: allow_quarantine) }
+
+    shared_examples 'a #list_all_commits message' do
+      it 'sends a list_all_commits message' do
+        expected_repository = repository.gitaly_repository.dup
+        expected_repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string)
+
+        expect_next_instance_of(Gitaly::CommitService::Stub) do |service|
+          expect(service).to receive(:list_all_commits)
+            .with(gitaly_request_with_params(repository: expected_repository), kind_of(Hash))
+            .and_return([Gitaly::ListAllCommitsResponse.new(commits: gitaly_commits)])
+        end
+
+        expect(subject).to eq(commits)
+      end
+    end
+
+    shared_examples 'a #list_commits message' do
+      it 'sends a list_commits message' do
+        expect_next_instance_of(Gitaly::CommitService::Stub) do |service|
+          expect(service).to receive(:list_commits)
+            .with(gitaly_request_with_params(revisions: revisions + %w[--not --all]), kind_of(Hash))
+            .and_return([Gitaly::ListCommitsResponse.new(commits: gitaly_commits)])
+        end
+
+        expect(subject).to eq(commits)
+      end
+    end
+
+    before do
+      ::Gitlab::GitalyClient.clear_stubs!
+
+      allow(Gitlab::Git::HookEnv)
+        .to receive(:all)
+        .with(repository.gl_repository)
+        .and_return(git_env)
+    end
+
+    context 'with hook environment' do
+      let(:git_env) do
+        {
+          'GIT_OBJECT_DIRECTORY_RELATIVE' => '.git/objects',
+          'GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE' => ['/dir/one', '/dir/two']
+        }
+      end
+
+      context 'with allowed quarantine' do
+        let(:allow_quarantine) { true }
+
+        it_behaves_like 'a #list_all_commits message'
+      end
+
+      context 'with disallowed quarantine' do
+        let(:allow_quarantine) { false }
+
+        it_behaves_like 'a #list_commits message'
+      end
+    end
+
+    context 'without hook environment' do
+      let(:git_env) do
+        {
+          'GIT_OBJECT_DIRECTORY_RELATIVE' => '',
+          'GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE' => []
+        }
+      end
+
+      context 'with allowed quarantine' do
+        let(:allow_quarantine) { true }
+
+        it_behaves_like 'a #list_commits message'
+      end
+
+      context 'with disallowed quarantine' do
+        let(:allow_quarantine) { false }
+
+        it_behaves_like 'a #list_commits message'
+      end
+    end
+  end
+
   describe '#commit_stats' do
     let(:request) do
       Gitaly::CommitStatsRequest.new(