Add latest changes from gitlab-org/gitlab@master

GITLAB_WORKHORSE_VERSION

@@ -1 +1 @@
-8.38.0
+8.39.0

app/assets/stylesheets/application.scss

@@ -53,7 +53,5 @@
 }
 
 /* Rules for overriding cloaking in startup-general.scss */
-.container-limited,
-.modal-dialog {
-  display: block;
-}
+@import 'startup/cloaking';
+@include cloak-startup-scss(block);

app/assets/stylesheets/startup/_cloaking.scss

@@ -0,0 +1,13 @@
+/**
+  Prevent flashing of content when using startup.css
+ */
+@mixin cloak-startup-scss($display) {
+  // Breadcrumbs and alerts on the top of the page
+  .content-wrapper > .alert-wrapper,
+  // Content on pages
+  #content-body,
+  // Prevent flashing of haml generated modal contents
+  .modal-dialog {
+    display: $display;
+  }
+}

app/graphql/resolvers/group_issues_resolver.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Resolvers
+  class GroupIssuesResolver < IssuesResolver
+    argument :include_subgroups, GraphQL::BOOLEAN_TYPE,
+             required: false,
+             default_value: false,
+             description: 'Include issues belonging to subgroups.'
+  end
+end

app/graphql/types/group_type.rb

@@ -47,7 +47,7 @@ module Types
           Types::IssueType.connection_type,
           null: true,
           description: 'Issues of the group',
-          resolver: Resolvers::IssuesResolver
+          resolver: Resolvers::GroupIssuesResolver
 
     field :milestones, Types::MilestoneType.connection_type, null: true,
           description: 'Milestones of the group',

app/models/ci/pipeline_artifact.rb

@@ -20,7 +20,7 @@ module Ci
     validates :pipeline, :project, :file_format, :file, presence: true
     validates :file_store, presence: true, inclusion: { in: FILE_STORE_SUPPORTED }
     validates :size, presence: true, numericality: { less_than_or_equal_to: FILE_SIZE_LIMIT }
-    validates :file_type, presence: true, uniqueness: { scope: [:pipeline_id] }
+    validates :file_type, presence: true
 
     enum file_type: {
       code_coverage: 1

app/views/profiles/two_factor_auths/_codes.html.haml

@@ -2,11 +2,11 @@
   - lose_2fa_message =  _('Should you ever lose your phone or access to your one time password secret, each of these recovery codes can be used one time each to regain access to your account. Please save them in a safe place, or you %{b_start}will%{b_end} lose access to your account.') % { b_start:'<b>', b_end:'</b>' }
   = lose_2fa_message.html_safe
 
-.codes.card
+.codes.card{ data: { qa_selector: 'codes_content' } }
   %ul
     - @codes.each do |code|
       %li
-        %span.monospace= code
+        %span.monospace{ data: { qa_selector: 'code_content' } }= code
 
 .d-flex
   = link_to _('Proceed'), profile_account_path, class: 'btn btn-success gl-mr-3', data: { qa_selector: 'proceed_button' }

changelogs/unreleased/229433-graphql-issue-subgroup.yml

@@ -0,0 +1,5 @@
+---
+title: 'GraphQL: Issues - Added ''include_subgroup'' parameter'
+merge_request: 39279
+author:
+type: added

changelogs/unreleased/230437-usage-data-optimize-counts-terraform_reports.yml

@@ -1,5 +0,0 @@
----
-title: Optimize counts.terraform_reports usage ping counter
-merge_request: 37498
-author:
-type: performance

changelogs/unreleased/improve-compliance-merge-request-query.yml

@@ -0,0 +1,5 @@
+---
+title: Add index for compliance merged MRs to events
+merge_request: 38885
+author:
+type: performance

changelogs/unreleased/sh-update-workhorse-8-39.yml

@@ -0,0 +1,5 @@
+---
+title: Fix CI job artifacts metadata not extracting on some S3 providers
+merge_request: 39345
+author:
+type: fixed

db/migrate/20200807070820_add_index_for_compliance_merged_merge_request_to_events.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddIndexForComplianceMergedMergeRequestToEvents < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_events_on_project_id_and_id_desc_on_merged_action'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :events, [:project_id, :id],
+      order: { id: :desc },
+      where: "action = 7", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :events, INDEX_NAME
+  end
+end

db/post_migrate/20200811211536_add_index_to_ci_job_artifacts_for_terraform_reports_id.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-class AddIndexToCiJobArtifactsForTerraformReportsId < ActiveRecord::Migration[6.0]
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-  INDEX_NAME = 'index_ci_job_artifacts_id_for_terraform_reports'
-
-  disable_ddl_transaction!
-
-  def up
-    add_concurrent_index :ci_job_artifacts, :id, where: 'file_type = 18', name: INDEX_NAME
-  end
-
-  def down
-    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
-  end
-end

db/schema_migrations/20200807070820

@@ -0,0 +1 @@
+a1fb6b5d486eee27aaed5083aa6340073bf05abab8f201fd9097374dbd345b90

db/schema_migrations/20200811211536

@@ -1 +0,0 @@
-d4ea24092289f6dba294c502b8ce89748165973fb2d7989fa7615433599a0c0c

db/structure.sql

@@ -19198,8 +19198,6 @@ CREATE UNIQUE INDEX index_ci_instance_variables_on_key ON public.ci_instance_var
 
 CREATE INDEX index_ci_job_artifacts_for_terraform_reports ON public.ci_job_artifacts USING btree (project_id, id) WHERE (file_type = 18);
 
-CREATE INDEX index_ci_job_artifacts_id_for_terraform_reports ON public.ci_job_artifacts USING btree (id) WHERE (file_type = 18);
-
 CREATE INDEX index_ci_job_artifacts_on_expire_at_and_job_id ON public.ci_job_artifacts USING btree (expire_at, job_id);
 
 CREATE INDEX index_ci_job_artifacts_on_file_store ON public.ci_job_artifacts USING btree (file_store);
@@ -19602,6 +19600,8 @@ CREATE INDEX index_events_on_project_id_and_created_at ON public.events USING bt
 
 CREATE INDEX index_events_on_project_id_and_id ON public.events USING btree (project_id, id);
 
+CREATE INDEX index_events_on_project_id_and_id_desc_on_merged_action ON public.events USING btree (project_id, id DESC) WHERE (action = 7);
+
 CREATE INDEX index_events_on_target_type_and_target_id ON public.events USING btree (target_type, target_id);
 
 CREATE UNIQUE INDEX index_events_on_target_type_and_target_id_and_fingerprint ON public.events USING btree (target_type, target_id, fingerprint);

doc/api/graphql/reference/gitlab_schema.graphql

@@ -5988,6 +5988,11 @@ type Group {
     """
     iids: [String!]
 
+    """
+    Include issues belonging to subgroups.
+    """
+    includeSubgroups: Boolean = false
+
     """
     Iterations applied to the issue
     """

doc/api/graphql/reference/gitlab_schema.json

@@ -16685,6 +16685,16 @@
                   },
                   "defaultValue": null
                 },
+                {
+                  "name": "includeSubgroups",
+                  "description": "Include issues belonging to subgroups.",
+                  "type": {
+                    "kind": "SCALAR",
+                    "name": "Boolean",
+                    "ofType": null
+                  },
+                  "defaultValue": "false"
+                },
                 {
                   "name": "after",
                   "description": "Returns the elements in the list that come after the specified cursor.",

doc/ci/variables/predefined_variables.md

@@ -48,6 +48,7 @@ Kubernetes-specific environment variables are detailed in the
 | `CI_CONFIG_PATH`                              | 9.4    | 0.5    | The path to CI configuration file. Defaults to `.gitlab-ci.yml`                                                                                                                                                                                                                                                                                                   |
 | `CI_DEBUG_TRACE`                              | all    | 1.7    | Whether [debug logging (tracing)](README.md#debug-logging) is enabled                                                                                                                                                                                                                                                                                      |
 | `CI_DEFAULT_BRANCH`                           | 12.4   | all    | The name of the default branch for the project.                                                                                                                                                                                                                                                                                                            |
+| `CI_DEPLOY_FREEZE`                            | 13.2   | all    | Included with the value `true` if the pipeline runs during a [deploy freeze window](../../user/project/releases/index.md#prevent-unintentional-releases-by-setting-a-deploy-freeze).                                                                                                                                                                                                                                    |
 | `CI_DEPLOY_PASSWORD`                          | 10.8   | all    | Authentication password of the [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token), only present if the Project has one related.                                                                                                                                                                                                                                    |
 | `CI_DEPLOY_USER`                              | 10.8   | all    | Authentication username of the [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token), only present if the Project has one related.                                                                                                                                                                                                                                    |
 | `CI_DISPOSABLE_ENVIRONMENT`                   | all    | 10.1   | Marks that the job is executed in a disposable environment (something that is created only for this job and disposed of/destroyed after the execution - all executors except `shell` and `ssh`). If the environment is disposable, it is set to true, otherwise it is not defined at all.                                                                  |

doc/development/documentation/styleguide.md

@@ -251,7 +251,7 @@ GitLab documentation should be clear and easy to understand.
 
 - Be clear, concise, and stick to the goal of the documentation.
 - Write in US English with US grammar. (Tested in [`British.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/British.yml).)
-- Use inclusive language.
+- Use [inclusive language](#inclusive-language).
 
 ### Point of view
 
@@ -313,6 +313,60 @@ Capitalize names of:
 
 Follow the capitalization style listed at the [authoritative source](#links-to-external-documentation) for the entity, which may use non-standard case styles. For example: GitLab and npm.
 
+### Inclusive language
+
+We strive to create documentation that is inclusive. This section includes guidance and examples in the
+following categories:
+
+- [Gender-specific wording](#avoid-gender-specific-wording).
+- [Ableist language](#avoid-ableist-language).
+- [Cultural sensitivity](#culturally-sensitive-language).
+
+We write our developer documentation with inclusivity and diversity in mind. This page is not an exhaustive reference, but describes some general guidelines and examples that illustrate some best practices to follow.
+
+#### Avoid gender-specific wording
+
+When possible, use gender-neutral pronouns. For example, you can use a singular
+[they](https://developers.google.com/style/pronouns#gender-neutral-pronouns) as a gender-neutral
+pronoun.
+
+Avoid the use of gender-specific pronouns, unless referring to a specific person.
+
+| Use                               | Avoid           |
+|-----------------------------------|-----------------|
+| People, humanity                  | Mankind         |
+| GitLab Team Members               | Manpower        |
+| You can install; They can install | He can install; She can install |
+
+If you need to set up [Fake user information](#fake-user-information), use diverse or non-gendered
+names with common surnames.
+
+#### Avoid ableist language
+
+Avoid terms that are also used in negative stereotypes for different groups.
+
+| Use                    | Avoid                |
+|------------------------|----------------------|
+| Check for completeness | Sanity check         |
+| Uncertain outliers     | Crazy outliers       |
+| Slows the service      | Cripples the service |
+| Placeholder variable   | Dummy variable       |
+| Active/Inactive        | Enabled/Disabled     |
+| On/Off                 | Enabled/Disabled     |
+
+Credit: [Avoid ableist language](https://developers.google.com/style/inclusive-documentation#ableist-language) in the Google Developer Style Guide.
+
+#### Culturally sensitive language
+
+Avoid terms that reflect negative cultural stereotypes and history. In most cases, you can replace terms such as `master` and `slave` with terms that are more precise and functional, such as `primary` and `secondary`.
+
+| Use                   | Avoid                |
+|-----------------------|----------------------|
+| Primary / secondary   | Master / slave       |
+| Blacklist / whitelist | Allowlist / denylist |
+
+For more information see the following [Internet Draft specification](https://tools.ietf.org/html/draft-knodel-terminology-02).
+
 ### Language to avoid
 
 When creating documentation, limit or avoid the use of the following verb

doc/user/markdown.md

@@ -348,10 +348,7 @@ The wrapping tags can be either curly braces or square brackets:
 - [- deletion 4 -]
 ```
 
-- {+ addition 1 +}
-- [+ addition 2 +]
-- {- deletion 3 -}
-- [- deletion 4 -]
+![Inline diff as rendered by GitLab's interface](img/inline_diff_01_v13_3.png)
 
 ---
 
@@ -373,9 +370,7 @@ backslash `\`, otherwise the diff highlight don't render correctly:
 - {+ Text with escaped \`backticks\` inside +}
 ```
 
-- {+ Just regular text +}
-- {+ Text with `backticks` inside +}
-- {+ Text with escaped \`backticks\` inside +}
+![Inline diff with mixed formatting, as rendered by GitLab's interface](img/inline_diff_02_v13_3.png)
 
 ### Math
 
@@ -475,16 +470,7 @@ unordered or ordered lists:
    1. [x] Sub-task 2
 ```
 
-- [x] Completed task
-- [ ] Incomplete task
-  - [ ] Sub-task 1
-  - [x] Sub-task 2
-  - [ ] Sub-task 3
-
-1. [x] Completed task
-1. [ ] Incomplete task
-   1. [ ] Sub-task 1
-   1. [x] Sub-task 2
+![A task list as rendered by GitLab's interface](img/completed_tasks_v13_3.png)
 
 ### Table of contents
 

lib/gitlab/ci/pipeline/metrics.rb

@@ -45,15 +45,6 @@ module Gitlab
             Gitlab::Metrics.counter(name, comment)
           end
         end
-
-        def pipelines_created_counter
-          strong_memoize(:pipelines_created_count) do
-            name = :pipelines_created_total
-            comment = 'Counter of pipelines created'
-
-            Gitlab::Metrics.counter(name, comment)
-          end
-        end
       end
     end
   end

qa/qa/page/profile/two_factor_auth.rb

@@ -16,6 +16,8 @@ module QA
 
         view 'app/views/profiles/two_factor_auths/_codes.html.haml' do
           element :proceed_button
+          element :codes_content
+          element :code_content
         end
 
         def click_configure_it_later_button
@@ -34,6 +36,13 @@ module QA
           click_element :register_2fa_app_button
         end
 
+        def recovery_codes
+          code_elements = within_element(:codes_content) do
+            all_elements(:code_content, minimum: 1)
+          end
+          code_elements.map { |code_content| code_content.text }
+        end
+
         def click_proceed_button
           click_element :proceed_button
         end

qa/qa/resource/group.rb

@@ -18,10 +18,12 @@ module QA
       attribute :id
       attribute :name
       attribute :runners_token
+      attribute :require_two_factor_authentication
 
       def initialize
         @path = Runtime::Namespace.name
         @description = "QA test run at #{Runtime::Namespace.time}"
+        @require_two_factor_authentication = false
       end
 
       def fabricate!
@@ -72,7 +74,8 @@ module QA
           parent_id: sandbox.id,
           path: path,
           name: path,
-          visibility: 'public'
+          visibility: 'public',
+          require_two_factor_authentication: @require_two_factor_authentication
         }
       end
 

qa/qa/specs/features/browser_ui/1_manage/login/2fa_recovery_spec.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module QA
+  context 'Manage', :requires_admin, :skip_live_env do
+    describe '2FA' do
+      let(:owner_user) do
+        Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_2fa_owner_username_1, Runtime::Env.gitlab_qa_2fa_owner_password_1)
+      end
+
+      let(:developer_user) do
+        Resource::User.fabricate_via_api! do |resource|
+          resource.api_client = admin_api_client
+        end
+      end
+
+      let(:sandbox_group) do
+        Resource::Sandbox.fabricate! do |sandbox_group|
+          sandbox_group.path = "gitlab-qa-2fa-recovery-sandbox-group-#{SecureRandom.hex(4)}"
+          sandbox_group.api_client = owner_api_client
+        end
+      end
+
+      let(:group) do
+        QA::Resource::Group.fabricate_via_api! do |group|
+          group.sandbox = sandbox_group
+          group.api_client = owner_api_client
+          group.require_two_factor_authentication = true
+        end
+      end
+
+      before do
+        group.add_member(developer_user, Resource::Members::AccessLevel::DEVELOPER)
+      end
+
+      it 'allows using 2FA recovery code once only' do
+        recovery_code = enable_2fa_for_user_and_fetch_recovery_code(developer_user)
+
+        Flow::Login.sign_in(as: developer_user, skip_page_validation: true)
+
+        Page::Main::TwoFactorAuth.perform do |two_fa_auth|
+          two_fa_auth.set_2fa_code(recovery_code)
+          two_fa_auth.click_verify_code_button
+        end
+
+        expect(Page::Main::Menu.perform(&:signed_in?)).to be_truthy
+
+        Page::Main::Menu.perform(&:sign_out)
+
+        Flow::Login.sign_in(as: developer_user, skip_page_validation: true)
+
+        Page::Main::TwoFactorAuth.perform do |two_fa_auth|
+          two_fa_auth.set_2fa_code(recovery_code)
+          two_fa_auth.click_verify_code_button
+        end
+
+        expect(page).to have_text('Invalid two-factor code')
+      end
+
+      after do
+        group.set_require_two_factor_authentication(value: 'false')
+        group.remove_via_api!
+        sandbox_group.remove_via_api!
+        developer_user.remove_via_api!
+      end
+
+      def admin_api_client
+        @admin_api_client ||= Runtime::API::Client.as_admin
+      end
+
+      def owner_api_client
+        @owner_api_client ||= Runtime::API::Client.new(:gitlab, user: owner_user)
+      end
+
+      def enable_2fa_for_user_and_fetch_recovery_code(user)
+        Flow::Login.while_signed_in(as: user) do
+          Page::Profile::TwoFactorAuth.perform do |two_fa_auth|
+            @otp = QA::Support::OTP.new(two_fa_auth.otp_secret_content)
+
+            two_fa_auth.set_pin_code(@otp.fresh_otp)
+            two_fa_auth.click_register_2fa_app_button
+
+            recovery_code = two_fa_auth.recovery_codes.sample
+
+            two_fa_auth.click_proceed_button
+
+            recovery_code
+          end
+        end
+      end
+    end
+  end
+end

spec/finders/autocomplete/move_to_project_finder_spec.rb

@@ -22,14 +22,14 @@ RSpec.describe Autocomplete::MoveToProjectFinder do
         expect(finder.execute).to be_empty
       end
 
-      it 'returns projects equal or above Gitlab::Access::REPORTER ordered by name' do
+      it 'returns projects equal or above Gitlab::Access::REPORTER' do
         reporter_project.add_reporter(user)
         developer_project.add_developer(user)
         maintainer_project.add_maintainer(user)
 
         finder = described_class.new(user, project_id: project.id)
 
-        expect(finder.execute.to_a).to eq([reporter_project, developer_project, maintainer_project])
+        expect(finder.execute.to_a).to contain_exactly(reporter_project, developer_project, maintainer_project)
       end
 
       it 'does not include the source project' do
@@ -88,10 +88,10 @@ RSpec.describe Autocomplete::MoveToProjectFinder do
         wadus_project.add_maintainer(user)
 
         expect(described_class.new(user, project_id: project.id).execute.to_a)
-          .to eq([foo_project, wadus_project])
+          .to contain_exactly(foo_project, wadus_project)
 
         expect(described_class.new(user, project_id: project.id, search: 'wadus').execute.to_a)
-          .to eq([wadus_project])
+          .to contain_exactly(wadus_project)
       end
 
       it 'allows searching by parent namespace' do

spec/graphql/resolvers/group_issues_resolver_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Resolvers::GroupIssuesResolver do
+  include GraphqlHelpers
+
+  let_it_be(:current_user) { create(:user) }
+
+  let_it_be(:group)         { create(:group) }
+  let_it_be(:project)       { create(:project, group: group) }
+  let_it_be(:other_project) { create(:project, group: group) }
+  let_it_be(:subgroup)      { create(:group, parent: group) }
+  let_it_be(:subproject)    { create(:project, group: subgroup) }
+
+  let_it_be(:issue1)    { create(:incident, project: project, state: :opened, created_at: 3.hours.ago, updated_at: 3.hours.ago) }
+  let_it_be(:issue2)    { create(:issue, project: project, state: :closed, title: 'foo', created_at: 1.hour.ago, updated_at: 1.hour.ago, closed_at: 1.hour.ago) }
+  let_it_be(:issue3)    { create(:issue, project: other_project, state: :closed, title: 'foo', created_at: 1.hour.ago, updated_at: 1.hour.ago, closed_at: 1.hour.ago) }
+  let_it_be(:issue4)    { create(:issue) }
+
+  let_it_be(:subissue1) { create(:issue, project: subproject) }
+  let_it_be(:subissue2) { create(:issue, project: subproject) }
+  let_it_be(:subissue3) { create(:issue, project: subproject) }
+
+  before_all do
+    group.add_developer(current_user)
+    subgroup.add_developer(current_user)
+  end
+
+  describe '#resolve' do
+    it 'finds all group issues' do
+      result = resolve(described_class, obj: group, ctx: { current_user: current_user })
+
+      expect(result).to contain_exactly(issue1, issue2, issue3)
+    end
+
+    it 'finds all group and subgroup issues' do
+      result = resolve(described_class, obj: group, args: { include_subgroups: true }, ctx: { current_user: current_user })
+
+      expect(result).to contain_exactly(issue1, issue2, issue3, subissue1, subissue2, subissue3)
+    end
+  end
+end

spec/models/ci/pipeline_artifact_spec.rb

@@ -19,7 +19,6 @@ RSpec.describe Ci::PipelineArtifact, type: :model do
     it { is_expected.to validate_presence_of(:file_format) }
     it { is_expected.to validate_presence_of(:size) }
     it { is_expected.to validate_presence_of(:file) }
-    it { is_expected.to validate_uniqueness_of(:file_type).scoped_to([:pipeline_id]).ignoring_case_sensitivity }
 
     context 'when attributes are valid' do
       it 'returns no errors' do