Add latest changes from gitlab-org/gitlab@master

db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class IndexJobArtifactsOnTraceTypeAndExpireAt < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at'
+  TIMESTAMPS = "'2021-04-22 00:00:00', '2021-05-22 00:00:00', '2021-06-22 00:00:00', '2022-01-22 00:00:00', '2022-02-22 00:00:00', '2022-03-22 00:00:00', '2022-04-22 00:00:00'"
+
+  def up
+    add_concurrent_index :ci_job_artifacts, :id, where: "file_type = 3 AND expire_at IN (#{TIMESTAMPS})", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME
+  end
+end

db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class ScheduleTraceExpiryRemoval < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'RemoveAllTraceExpirationDates'
+  BATCH_SIZE = 100_000
+  DELAY_INTERVAL = 4.minutes
+
+  disable_ddl_transaction!
+
+  # Stubbed class to connect to the CI database
+  # connects_to has to be called in abstract classes.
+  class MultiDbAdaptableClass < ActiveRecord::Base
+    self.abstract_class = true
+
+    if Gitlab::Database.has_config?(:ci)
+      connects_to database: { writing: :ci, reading: :ci }
+    end
+  end
+
+  # Stubbed class to access the ci_job_artifacts table
+  class JobArtifact < MultiDbAdaptableClass
+    include EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    TARGET_TIMESTAMPS = [
+      Date.new(2021, 04, 22).midnight.utc,
+      Date.new(2021, 05, 22).midnight.utc,
+      Date.new(2021, 06, 22).midnight.utc,
+      Date.new(2022, 01, 22).midnight.utc,
+      Date.new(2022, 02, 22).midnight.utc,
+      Date.new(2022, 03, 22).midnight.utc,
+      Date.new(2022, 04, 22).midnight.utc
+    ].freeze
+
+    scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) }
+    scope :traces, -> { where(file_type: 3) }
+  end
+
+  def up
+    return unless Gitlab.com?
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      JobArtifact.traces.in_targeted_timestamps,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20220131000000

@@ -0,0 +1 @@
+08326048e15f368f09bc10ebf5bee3e77e8b43813f66c19d24731497ca6a8485

db/schema_migrations/20220131000001

@@ -0,0 +1 @@
+59fe701bcaa102b7e0c1496198fa4aeea6b2e59132c951d1c9d54562c5e3900e

db/structure.sql

@@ -29522,6 +29522,8 @@ CREATE INDEX tmp_gitlab_subscriptions_max_seats_used_migration_2 ON gitlab_subsc
 
 CREATE INDEX tmp_idx_vulnerability_occurrences_on_id_where_report_type_7_99 ON vulnerability_occurrences USING btree (id) WHERE (report_type = ANY (ARRAY[7, 99]));
 
+CREATE INDEX tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at ON ci_job_artifacts USING btree (id) WHERE ((file_type = 3) AND (expire_at = ANY (ARRAY['2021-04-22 00:00:00+00'::timestamp with time zone, '2021-05-22 00:00:00+00'::timestamp with time zone, '2021-06-22 00:00:00+00'::timestamp with time zone, '2022-01-22 00:00:00+00'::timestamp with time zone, '2022-02-22 00:00:00+00'::timestamp with time zone, '2022-03-22 00:00:00+00'::timestamp with time zone, '2022-04-22 00:00:00+00'::timestamp with time zone])));
+
 CREATE INDEX tmp_index_container_repositories_on_id_migration_state ON container_repositories USING btree (id, migration_state);
 
 CREATE INDEX tmp_index_for_namespace_id_migration_on_group_members ON members USING btree (id) WHERE ((member_namespace_id IS NULL) AND ((type)::text = 'GroupMember'::text));

doc/administration/troubleshooting/linux_cheat_sheet.md

@@ -14,7 +14,7 @@ having an issue with GitLab, you may want to check your [support options](https:
 first, before attempting to use this information.
 
 WARNING:
-If you administer GitLab you are expected to know these commands for your distribution
+It is [beyond the scope of GitLab Support to assist in systems administration](https://about.gitlab.com/support/statement-of-support.html#training). GitLab administrators are expected to know these commands for their distribution
 of choice. If you are a GitLab Support Engineer, consider this a cross-reference to
 translate `yum` -> `apt-get` and the like.
 

doc/development/integrations/jira_connect.md

@@ -65,3 +65,12 @@ If the app install failed, you might need to delete `jira_connect_installations`
 
 1. Open the [database console](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#access-postgresql).
 1. Run `TRUNCATE TABLE jira_connect_installations CASCADE;`.
+
+#### Not authorized to access the file
+
+If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK's port public by following these steps:
+
+1. Open your GitLab workspace in Gitpod.
+1. When the GDK is running, select **Ports** in the bottom-right corner.
+1. On the left sidebar, select the port the GDK is listening to (typically `3000`).
+1. If the port is marked as private, select the lock icon to make it public.

doc/user/project/integrations/webhook_events.md

@@ -771,7 +771,8 @@ Payload example:
 Merge request events are triggered when:
 
 - A new merge request is created.
-- An existing merge request is updated, approved, unapproved, merged, or closed.
+- An existing merge request is updated, approved (by all required approvers), unapproved, merged, or closed.
+- An individual user adds or removes their approval to an existing merge request.
 - A commit is added in the source branch.
 - All threads are resolved on the merge request.
 
@@ -783,6 +784,8 @@ The available values for `object_attributes.action` in the payload are:
 - `update`
 - `approved`
 - `unapproved`
+- `approval`
+- `unapproval`
 - `merge`
 
 Request header:

lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Removing expire_at timestamps that shouldn't have
+    # been written to traces on gitlab.com.
+    class RemoveAllTraceExpirationDates
+      include Gitlab::Database::MigrationHelpers
+
+      BATCH_SIZE = 1_000
+
+      # Stubbed class to connect to the CI database
+      # connects_to has to be called in abstract classes.
+      class MultiDbAdaptableClass < ActiveRecord::Base
+        self.abstract_class = true
+
+        if Gitlab::Database.has_config?(:ci)
+          connects_to database: { writing: :ci, reading: :ci }
+        end
+      end
+
+      # Stubbed class to access the ci_job_artifacts table
+      class JobArtifact < MultiDbAdaptableClass
+        include EachBatch
+
+        self.table_name = 'ci_job_artifacts'
+
+        TARGET_TIMESTAMPS = [
+          Date.new(2021, 04, 22).midnight.utc,
+          Date.new(2021, 05, 22).midnight.utc,
+          Date.new(2021, 06, 22).midnight.utc,
+          Date.new(2022, 01, 22).midnight.utc,
+          Date.new(2022, 02, 22).midnight.utc,
+          Date.new(2022, 03, 22).midnight.utc,
+          Date.new(2022, 04, 22).midnight.utc
+        ].freeze
+
+        scope :traces, -> { where(file_type: 3) }
+        scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
+        scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) }
+      end
+
+      def perform(start_id, end_id)
+        return unless Gitlab.com?
+
+        JobArtifact.traces
+          .between(start_id, end_id)
+          .in_targeted_timestamps
+          .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) }
+      end
+    end
+  end
+end

lib/gitlab/cycle_analytics/summary/base.rb

@@ -4,27 +4,13 @@ module Gitlab
   module CycleAnalytics
     module Summary
       class Base
+        include Gitlab::CycleAnalytics::Summary::Defaults
+
         def initialize(project:, options:)
           @project = project
           @options = options
         end
 
-        def identifier
-          self.class.name.demodulize.underscore.to_sym
-        end
-
-        def title
-          raise NotImplementedError, "Expected #{self.name} to implement title"
-        end
-
-        def value
-          raise NotImplementedError, "Expected #{self.name} to implement value"
-        end
-
-        def links
-          []
-        end
-
         private
 
         attr_reader :project, :options

lib/gitlab/cycle_analytics/summary/defaults.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module Summary
+      module Defaults
+        def identifier
+          self.class.name.demodulize.underscore.to_sym
+        end
+
+        # :nocov: the class including this concern is expected to test this method.
+        def title
+          raise NotImplementedError, "Expected #{self.name} to implement title"
+        end
+        # :nocov:
+
+        # :nocov: the class including this concern is expected to test this method.
+        def value
+          raise NotImplementedError, "Expected #{self.name} to implement value"
+        end
+        # :nocov:
+
+        def links
+          []
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RemoveAllTraceExpirationDates, :migration, schema: 20220131000001 do
+  subject(:perform) { migration.perform(1, 99) }
+
+  let(:migration) { described_class.new }
+
+  let(:trace_in_range)         { create_trace!(id: 10,   created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) }
+  let(:trace_outside_range)    { create_trace!(id: 40,   created_at: Date.new(2020, 06, 22), expire_at: Date.new(2021, 01, 22)) }
+  let(:trace_without_expiry)   { create_trace!(id: 30,   created_at: Date.new(2020, 06, 21), expire_at: nil) }
+  let(:archive_in_range)       { create_archive!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) }
+  let(:trace_outside_id_range) { create_trace!(id: 100,  created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) }
+
+  before do
+    table(:namespaces).create!(id: 1, name: 'the-namespace', path: 'the-path')
+    table(:projects).create!(id: 1, name: 'the-project', namespace_id: 1)
+    table(:ci_builds).create!(id: 1, allow_failure: false)
+  end
+
+  context 'for self-hosted instances' do
+    it 'sets expire_at for artifacts in range to nil' do
+      expect { perform }.not_to change { trace_in_range.reload.expire_at }
+    end
+
+    it 'does not change expire_at timestamps that are not set to midnight' do
+      expect { perform }.not_to change { trace_outside_range.reload.expire_at }
+    end
+
+    it 'does not change expire_at timestamps that are set to midnight on a day other than the 22nd' do
+      expect { perform }.not_to change { trace_without_expiry.reload.expire_at }
+    end
+
+    it 'does not touch artifacts outside id range' do
+      expect { perform }.not_to change { archive_in_range.reload.expire_at }
+    end
+
+    it 'does not touch artifacts outside date range' do
+      expect { perform }.not_to change { trace_outside_id_range.reload.expire_at }
+    end
+  end
+
+  private
+
+  def create_trace!(**args)
+    table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 3)
+  end
+
+  def create_archive!(**args)
+    table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 1)
+  end
+end