Add latest changes from gitlab-org/gitlab@master

.rubocop_manual_todo.yml

@@ -177,10 +177,6 @@ Rails/SaveBang:
     - 'spec/controllers/sent_notifications_controller_spec.rb'
     - 'spec/controllers/sessions_controller_spec.rb'
     - 'spec/factories_spec.rb'
-    - 'spec/features/dashboard/datetime_on_tooltips_spec.rb'
-    - 'spec/features/dashboard/issuables_counter_spec.rb'
-    - 'spec/features/dashboard/project_member_activity_index_spec.rb'
-    - 'spec/features/dashboard/projects_spec.rb'
     - 'spec/frontend/fixtures/issues.rb'
     - 'spec/frontend/fixtures/merge_requests.rb'
     - 'spec/graphql/mutations/merge_requests/set_locked_spec.rb'

app/assets/javascripts/pipelines/components/pipelines_list/time_ago.vue

@@ -25,6 +25,9 @@ export default {
     skipped() {
       return this.pipeline?.details?.status?.label === 'skipped';
     },
+    stuck() {
+      return this.pipeline.flags.stuck;
+    },
     durationFormatted() {
       const date = new Date(this.duration * 1000);
 
@@ -67,7 +70,20 @@ export default {
     </div>
     <div :class="legacyTableMobileClass">
       <span v-if="showInProgress" data-testid="pipeline-in-progress">
-        <gl-icon name="hourglass" class="gl-vertical-align-baseline! gl-mr-2" :size="12" />
+        <gl-icon
+          v-if="stuck"
+          name="warning"
+          class="gl-mr-2"
+          :size="12"
+          data-testid="warning-icon"
+        />
+        <gl-icon
+          v-else
+          name="hourglass"
+          class="gl-vertical-align-baseline! gl-mr-2"
+          :size="12"
+          data-testid="hourglass-icon"
+        />
         {{ s__('Pipeline|In progress') }}
       </span>
 

app/models/user_detail.rb

@@ -19,7 +19,7 @@ class UserDetail < ApplicationRecord
 
   # For backward compatibility.
   # Older migrations (and their tests) reference the `User.migration_bot` where the `bio` attribute is set.
-  # Here we disable writing the markdown cache when the `bio_html` column does not exists.
+  # Here we disable writing the markdown cache when the `bio_html` column does not exist.
   override :invalidated_markdown_cache?
   def invalidated_markdown_cache?
     self.class.column_names.include?('bio_html') && super

changelogs/unreleased/pb-stuck-job-in-progress-ux.yml

@@ -0,0 +1,5 @@
+---
+title: Add warning icon beside in progress text if pipeline is stuck
+merge_request: 58427
+author:
+type: changed

changelogs/unreleased/rails-save-bang-features-dashboard.yml

@@ -0,0 +1,5 @@
+---
+title: Fix Rails/SaveBang rubocop offenses in spec/features/dashboard
+merge_request: 57898
+author: Abdul Wadood @abdulwd
+type: fixed

config/feature_flags/ops/usage_data_non_sql_metrics.yml

@@ -0,0 +1,8 @@
+---
+name: usage_data_non_sql_metrics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57050
+rollout_issue_url: 
+milestone: '13.11'
+type: ops
+group: group::product intelligence
+default_enabled: false

doc/administration/gitaly/praefect.md

@@ -1296,21 +1296,35 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t
 
 ## Migrate existing repositories to Gitaly Cluster
 
-If your GitLab instance already has repositories on single Gitaly nodes, these aren't migrated to
-Gitaly Cluster automatically.
+To migrate to Gitaly Cluster, existing repositories stored outside Gitaly Cluster must be
+moved. There is no automatic migration but the moves can be scheduled with the GitLab API.
 
-Project repositories may be moved from one storage location using the [Project repository storage moves API](../../api/project_repository_storage_moves.md). Note that this API cannot move all repository types. For moving other repositories types, see:
+GitLab repositories can be associated with projects, groups, and snippets. Each of these types
+have a separate API to schedule the respective repositories to move. To move all repositories
+on a GitLab instance, each of these types must be scheduled to move for each storage.
 
-- [Snippet repository storage moves API](../../api/snippet_repository_storage_moves.md).
-- [Group repository storage moves API](../../api/group_repository_storage_moves.md).
+Each repository is made read only when the move is scheduled. The repository is not writable
+until the move has completed.
 
-To move repositories to Gitaly Cluster:
+After creating and configuring Gitaly Cluster:
+
+1. Ensure all storages are accessible to the GitLab instance. In this example, these
+are `<original_storage_name>` and `<cluster_storage_name>`.
+1. [Configure repository storage weights](../repository_storage_paths.md#configure-where-new-repositories-are-stored)
+   so that the Gitaly Cluster receives all new projects. This stops new projects being created
+   on existing Gitaly nodes while the migration is in progress.
+1. Schedule repository moves for:
+   - [Projects](#bulk-schedule-projects).
+   - [Snippets](#bulk-schedule-snippets).
+   - [Groups](#bulk-schedule-groups). **(PREMIUM SELF)**
+
+### Bulk schedule projects
 
 1. [Schedule repository storage moves for all projects on a storage shard](../../api/project_repository_storage_moves.md#schedule-repository-storage-moves-for-all-projects-on-a-storage-shard) using the API. For example:
 
    ```shell
    curl --request POST --header "Private-Token: <your_access_token>" --header "Content-Type: application/json" \
-   --data '{"source_storage_name":"gitaly","destination_storage_name":"praefect"}' "https://gitlab.example.com/api/v4/project_repository_storage_moves"
+   --data '{"source_storage_name":"<original_storage_name>","destination_storage_name":"<cluster_storage_name>"}' "https://gitlab.example.com/api/v4/project_repository_storage_moves"
    ```
 
 1. [Query the most recent repository moves](../../api/project_repository_storage_moves.md#retrieve-all-project-repository-storage-moves)
@@ -1323,9 +1337,69 @@ To move repositories to Gitaly Cluster:
    using the API to confirm that all projects have moved. No projects should be returned
    with `repository_storage` field set to the old storage.
 
-In a similar way, you can move other repository types by using the
-[Snippet repository storage moves API](../../api/snippet_repository_storage_moves.md) **(FREE SELF)**
-or the [Groups repository storage moves API](../../api/group_repository_storage_moves.md) **(PREMIUM SELF)**.
+   ```shell
+   curl --header "Private-Token: <your_access_token>" --header "Content-Type: application/json" \
+   "https://gitlab.example.com/api/v4/projects?repository_storage=<original_storage_name>"
+   ```
+
+   Alternatively use [the rails console](../operations/rails_console.md) to
+   confirm that all projects have moved. Run the following in the rails console:
+
+   ```ruby
+   ProjectRepository.for_repository_storage('<original_storage_name>')
+   ```
+
+1. Repeat for each storage as required.
+
+### Bulk schedule snippets
+
+1. [Schedule repository storage moves for all snippets on a storage shard](../../api/snippet_repository_storage_moves.md#schedule-repository-storage-moves-for-all-snippets-on-a-storage-shard) using the API. For example:
+
+   ```shell
+   curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" \
+   --data '{"source_storage_name":"<original_storage_name>","destination_storage_name":"<cluster_storage_name>"}' "https://gitlab.example.com/api/v4/snippet_repository_storage_moves"
+   ```
+
+1. [Query the most recent repository moves](../../api/snippet_repository_storage_moves.md#retrieve-all-snippet-repository-storage-moves)
+   using the API. The query indicates either:
+   - The moves have completed successfully. The `state` field is `finished`.
+   - The moves are in progress. Re-query the repository move until it completes successfully.
+   - The moves have failed. Most failures are temporary and are solved by rescheduling the move.
+
+1. After the moves are complete, use [the rails console](../operations/rails_console.md) to
+   confirm that all snippets have moved. No snippets should be returned for the original
+   storage. Run the following in the rails console:
+
+   ```ruby
+   SnippetRepository.for_repository_storage('<original_storage_name>')
+   ```
+
+1. Repeat for each storage as required.
+
+### Bulk schedule groups **(PREMIUM SELF)**
+
+1. [Schedule repository storage moves for all groups on a storage shard](../../api/group_repository_storage_moves.md#schedule-repository-storage-moves-for-all-groups-on-a-storage-shard) using the API.
+
+    ```shell
+    curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" \
+    --data '{"source_storage_name":"<original_storage_name>","destination_storage_name":"<cluster_storage_name>"}' "https://gitlab.example.com/api/v4/group_repository_storage_moves"
+    ```
+
+1. [Query the most recent repository moves](../../api/group_repository_storage_moves.md#retrieve-all-group-repository-storage-moves)
+   using the API. The query indicates either:
+   - The moves have completed successfully. The `state` field is `finished`.
+   - The moves are in progress. Re-query the repository move until it completes successfully.
+   - The moves have failed. Most failures are temporary and are solved by rescheduling the move.
+
+1. After the moves are complete, use [the rails console](../operations/rails_console.md) to
+   confirm that all groups have moved. No groups should be returned for the original
+   storage. Run the following in the rails console:
+
+   ```ruby
+   GroupWikiRepository.for_repository_storage('<original_storage_name>')
+   ```
+
+1. Repeat for each storage as required.
 
 ## Debugging Praefect
 

doc/development/database/add_foreign_key_to_existing_column.md

@@ -58,6 +58,9 @@ emails = Email.where(user_id: 1) # returns emails for the deleted user
 
 Add a `NOT VALID` foreign key constraint to the table, which enforces consistency on the record changes.
 
+[Using the `with_lock_retries` helper method is advised when performing operations on high-traffic tables](../migration_style_guide.md#when-to-use-the-helper-method),
+in this case, if the table or the foreign table is a high-traffic table, we should use the helper method.
+
 In the example above, you'd be still able to update records in the `emails` table. However, when you'd try to update the `user_id` with non-existent value, the constraint causes a database error.
 
 Migration file for adding `NOT VALID` foreign key:

doc/user/application_security/vulnerability_report/index.md

@@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 # Vulnerability Report **(ULTIMATE)**
 
 The Vulnerability Report provides information about vulnerabilities from scans of the branch most
-recently merged into the default branch. It is available at the instance, group, and project level.
+recently merged into the default branch. It is available for groups, projects, and the Security Center.
 
 At all levels, the Vulnerability Report contains:
 
@@ -73,7 +73,7 @@ The content of the Project filter depends on the current level:
 
 | Level          | Content of the Project filter |
 |:---------------|:------------------------------|
-| Instance level | Only projects you've [added to the instance-level Security Center](../security_dashboard/index.md#adding-projects-to-the-security-center). |
+| Security Center | Only projects you've [added to your personal Security Center](../security_dashboard/index.md#adding-projects-to-the-security-center). |
 | Group level    | All projects in the group. |
 | Project level  | Not applicable. |
 

lib/api/api.rb

@@ -294,6 +294,7 @@ module API
       mount ::API::Unleash
       mount ::API::UsageData
       mount ::API::UsageDataQueries
+      mount ::API::UsageDataNonSqlMetrics
       mount ::API::UserCounts
       mount ::API::Users
       mount ::API::Variables

lib/api/usage_data_non_sql_metrics.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+  class UsageDataNonSqlMetrics < ::API::Base
+    before { authenticated_as_admin! }
+
+    feature_category :usage_ping
+
+    namespace 'usage_data' do
+      before do
+        not_found! unless Feature.enabled?(:usage_data_non_sql_metrics, default_enabled: :yaml, type: :ops)
+      end
+
+      desc 'Get Non SQL usage ping metrics' do
+        detail 'This feature was introduced in GitLab 13.11.'
+      end
+
+      get 'non_sql_metrics' do
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')
+
+        data = Gitlab::UsageDataNonSqlMetrics.uncached_data
+
+        present data
+      end
+    end
+  end
+end

lib/api/usage_data_queries.rb

@@ -8,7 +8,7 @@ module API
 
     namespace 'usage_data' do
       before do
-        not_found! unless Feature.enabled?(:usage_data_queries_api, default_enabled: false, type: :ops)
+        not_found! unless Feature.enabled?(:usage_data_queries_api, default_enabled: :yaml, type: :ops)
       end
 
       desc 'Get raw SQL queries for usage data SQL metrics' do

qa/qa.rb

@@ -98,6 +98,7 @@ module QA
     autoload :Design, 'qa/resource/design'
     autoload :RegistryRepository, 'qa/resource/registry_repository'
     autoload :Package, 'qa/resource/package'
+    autoload :PipelineSchedules, 'qa/resource/pipeline_schedules'
 
     module KubernetesCluster
       autoload :Base, 'qa/resource/kubernetes_cluster/base'

qa/qa/resource/pipeline_schedules.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module QA
+  module Resource
+    class PipelineSchedules < Base
+      attribute :id
+      attribute :ref
+      attribute :description
+
+      # Cron schedule form "* * * * *"
+      # String of integers in order of "minute hour day-of-month month day-of-week"
+      attribute :cron
+
+      attribute :project do
+        Resource::Project.fabricate! do |project|
+          project.name = 'project-with-pipeline-schedule'
+        end
+      end
+
+      def initialize
+        @cron = '0 * * * *' # default to schedule at the beginning of the hour
+        @description = 'QA test scheduling pipeline.'
+        @ref = project.default_branch
+      end
+
+      def api_get_path
+        "/projects/#{project.id}/pipeline_schedules/#{id}"
+      end
+
+      def api_post_path
+        "/projects/#{project.id}/pipeline_schedules"
+      end
+
+      def api_post_body
+        {
+          description: description,
+          ref: ref,
+          cron: cron
+        }
+      end
+
+      private
+
+      def resource_web_url(resource)
+        resource = resource.has_key?(:owner) ? resource.fetch(:owner) : resource
+        super
+      end
+    end
+  end
+end

qa/qa/resource/project.rb

@@ -179,6 +179,10 @@ module QA
         "#{api_get_path}/pipelines"
       end
 
+      def api_pipeline_schedules_path
+        "#{api_get_path}/pipeline_schedules"
+      end
+
       def api_put_path
         "/projects/#{id}"
       end
@@ -290,6 +294,10 @@ module QA
         parse_body(get(Runtime::API::Request.new(api_client, api_pipelines_path).url))
       end
 
+      def pipeline_schedules
+        parse_body(get(Runtime::API::Request.new(api_client, api_pipeline_schedules_path).url))
+      end
+
       private
 
       def transform_api_resource(api_resource)

qa/qa/resource/user.rb

@@ -118,6 +118,10 @@ module QA
         '/users'
       end
 
+      def api_block_path
+        "/users/#{id}/block"
+      end
+
       def api_post_body
         {
           admin: admin,
@@ -143,6 +147,14 @@ module QA
         end
       end
 
+      def block!
+        response = post(Runtime::API::Request.new(api_client, api_block_path).url, nil)
+
+        unless response.code == HTTP_STATUS_CREATED
+          raise ResourceUpdateFailedError, "Failed to block user. Request returned (#{response.code}): `#{response}`."
+        end
+      end
+
       private
 
       def ldap_post_body

qa/qa/specs/features/api/4_verify/cancel_pipeline_when_block_user_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module QA
+  RSpec.describe 'Verify', :requires_admin do
+    describe 'When user is blocked' do
+      let!(:admin_api_client) { Runtime::API::Client.as_admin }
+      let!(:user_api_client) { Runtime::API::Client.new(:gitlab, user: user) }
+
+      let(:user) do
+        Resource::User.fabricate_via_api! do |resource|
+          resource.api_client = admin_api_client
+        end
+      end
+
+      let(:project) do
+        Resource::Project.fabricate_via_api! do |project|
+          project.name = 'project-for-canceled-schedule'
+        end
+      end
+
+      before do
+        project.add_member(user, Resource::Members::AccessLevel::MAINTAINER)
+
+        Resource::PipelineSchedules.fabricate_via_api! do |schedule|
+          schedule.api_client = user_api_client
+          schedule.project = project
+        end
+
+        Support::Waiter.wait_until { !pipeline_schedule[:id].nil? && pipeline_schedule[:active] == true }
+      end
+
+      after do
+        user.remove_via_api!
+        project.remove_via_api!
+      end
+
+      it 'pipeline schedule is canceled', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1730' do
+        user.block!
+
+        expect(pipeline_schedule[:active]).not_to be_truthy, "Expected schedule active state to be false - active state #{pipeline_schedule[:active]}"
+      end
+
+      private
+
+      def pipeline_schedule
+        project.pipeline_schedules.first
+      end
+    end
+  end
+end

spec/features/dashboard/datetime_on_tooltips_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe 'Tooltips on .timeago dates', :js do
 
   context 'on the activity tab' do
     before do
-      Event.create( project: project, author_id: user.id, action: :joined,
+      Event.create!( project: project, author_id: user.id, action: :joined,
                     updated_at: created_date, created_at: created_date)
 
       sign_in user

spec/features/dashboard/issuables_counter_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe 'Navigation bar counter', :use_clean_rails_memory_store_caching d
 
   before do
     issue.assignees = [user]
-    merge_request.update(assignees: [user])
+    merge_request.update!(assignees: [user])
     sign_in(user)
   end
 
@@ -35,7 +35,7 @@ RSpec.describe 'Navigation bar counter', :use_clean_rails_memory_store_caching d
 
     expect_counters('merge_requests', '1')
 
-    merge_request.update(assignees: [])
+    merge_request.update!(assignees: [])
 
     user.invalidate_cache_counts
 

spec/features/dashboard/project_member_activity_index_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe 'Project member activity', :js do
   end
 
   def visit_activities_and_wait_with_event(event_type)
-    Event.create(project: project, author_id: user.id, action: event_type)
+    Event.create!(project: project, author_id: user.id, action: event_type)
     visit activity_project_path(project)
     wait_for_requests
   end

spec/features/dashboard/projects_spec.rb

@@ -41,7 +41,7 @@ RSpec.describe 'Dashboard Projects' do
           expect(page).to have_content('Developer')
         end
 
-        project.members.last.update(access_level: 40)
+        project.members.last.update!(access_level: 40)
 
         visit dashboard_projects_path
 
@@ -183,7 +183,7 @@ RSpec.describe 'Dashboard Projects' do
       let(:guest_user) { create(:user) }
 
       before do
-        project.update(public_builds: false)
+        project.update!(public_builds: false)
         project.add_guest(guest_user)
         sign_in(guest_user)
       end

spec/frontend/pipelines/time_ago_spec.js

@@ -1,25 +1,33 @@
 import { GlIcon } from '@gitlab/ui';
 import { shallowMount } from '@vue/test-utils';
+import { extendedWrapper } from 'helpers/vue_test_utils_helper';
 import TimeAgo from '~/pipelines/components/pipelines_list/time_ago.vue';
 
 describe('Timeago component', () => {
   let wrapper;
 
-  const createComponent = (props = {}) => {
-    wrapper = shallowMount(TimeAgo, {
-      propsData: {
-        pipeline: {
-          details: {
-            ...props,
+  const defaultProps = { duration: 0, finished_at: '' };
+
+  const createComponent = (props = defaultProps, stuck = false) => {
+    wrapper = extendedWrapper(
+      shallowMount(TimeAgo, {
+        propsData: {
+          pipeline: {
+            details: {
+              ...props,
+            },
+            flags: {
+              stuck,
+            },
           },
         },
-      },
-      data() {
-        return {
-          iconTimerSvg: `<svg></svg>`,
-        };
-      },
-    });
+        data() {
+          return {
+            iconTimerSvg: `<svg></svg>`,
+          };
+        },
+      }),
+    );
   };
 
   afterEach(() => {
@@ -29,8 +37,10 @@ describe('Timeago component', () => {
 
   const duration = () => wrapper.find('.duration');
   const finishedAt = () => wrapper.find('.finished-at');
-  const findInProgress = () => wrapper.find('[data-testid="pipeline-in-progress"]');
-  const findSkipped = () => wrapper.find('[data-testid="pipeline-skipped"]');
+  const findInProgress = () => wrapper.findByTestId('pipeline-in-progress');
+  const findSkipped = () => wrapper.findByTestId('pipeline-skipped');
+  const findHourGlassIcon = () => wrapper.findByTestId('hourglass-icon');
+  const findWarningIcon = () => wrapper.findByTestId('warning-icon');
 
   describe('with duration', () => {
     beforeEach(() => {
@@ -47,7 +57,7 @@ describe('Timeago component', () => {
 
   describe('without duration', () => {
     beforeEach(() => {
-      createComponent({ duration: 0, finished_at: '' });
+      createComponent();
     });
 
     it('should not render duration and timer svg', () => {
@@ -72,7 +82,7 @@ describe('Timeago component', () => {
 
   describe('without finishedTime', () => {
     beforeEach(() => {
-      createComponent({ duration: 0, finished_at: '' });
+      createComponent();
     });
 
     it('should not render time and calendar icon', () => {
@@ -99,6 +109,15 @@ describe('Timeago component', () => {
         expect(findSkipped().exists()).toBe(false);
       },
     );
+
+    it('should show warning icon beside in progress if pipeline is stuck', () => {
+      const stuck = true;
+
+      createComponent(defaultProps, stuck);
+
+      expect(findWarningIcon().exists()).toBe(true);
+      expect(findHourGlassIcon().exists()).toBe(false);
+    });
   });
 
   describe('skipped', () => {

spec/requests/api/usage_data_non_sql_metrics_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::UsageDataNonSqlMetrics do
+  include UsageDataHelpers
+
+  let_it_be(:admin) { create(:user, admin: true) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    stub_usage_data_connections
+  end
+
+  describe 'GET /usage_data/non_sql_metrics' do
+    let(:endpoint) { '/usage_data/non_sql_metrics' }
+
+    context 'with authentication' do
+      before do
+        stub_feature_flags(usage_data_non_sql_metrics: true)
+      end
+
+      it 'returns non sql metrics if user is admin' do
+        get api(endpoint, admin)
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response['counts']).to be_a(Hash)
+      end
+
+      it 'returns forbidden if user is not admin' do
+        get api(endpoint, user)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'without authentication' do
+      before do
+        stub_feature_flags(usage_data_non_sql_metrics: true)
+      end
+
+      it 'returns unauthorized' do
+        get api(endpoint)
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+      end
+    end
+
+    context 'when feature_flag is disabled' do
+      before do
+        stub_feature_flags(usage_data_non_sql_metrics: false)
+      end
+
+      it 'returns not_found for admin' do
+        get api(endpoint, admin)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+
+      it 'returns forbidden for non-admin' do
+        get api(endpoint, user)
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+  end
+end