diff --git a/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb b/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb new file mode 100644 index 00000000000..f47c04dd2ef --- /dev/null +++ b/db/post_migrate/20220131000000_index_job_artifacts_on_trace_type_and_expire_at.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexJobArtifactsOnTraceTypeAndExpireAt < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at' + TIMESTAMPS = "'2021-04-22 00:00:00', '2021-05-22 00:00:00', '2021-06-22 00:00:00', '2022-01-22 00:00:00', '2022-02-22 00:00:00', '2022-03-22 00:00:00', '2022-04-22 00:00:00'" + + def up + add_concurrent_index :ci_job_artifacts, :id, where: "file_type = 3 AND expire_at IN (#{TIMESTAMPS})", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb b/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb new file mode 100644 index 00000000000..8e282a9b8c2 --- /dev/null +++ b/db/post_migrate/20220131000001_schedule_trace_expiry_removal.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class ScheduleTraceExpiryRemoval < Gitlab::Database::Migration[1.0] + MIGRATION = 'RemoveAllTraceExpirationDates' + BATCH_SIZE = 100_000 + DELAY_INTERVAL = 4.minutes + + disable_ddl_transaction! + + # Stubbed class to connect to the CI database + # connects_to has to be called in abstract classes. + class MultiDbAdaptableClass < ActiveRecord::Base + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end + + # Stubbed class to access the ci_job_artifacts table + class JobArtifact < MultiDbAdaptableClass + include EachBatch + + self.table_name = 'ci_job_artifacts' + + TARGET_TIMESTAMPS = [ + Date.new(2021, 04, 22).midnight.utc, + Date.new(2021, 05, 22).midnight.utc, + Date.new(2021, 06, 22).midnight.utc, + Date.new(2022, 01, 22).midnight.utc, + Date.new(2022, 02, 22).midnight.utc, + Date.new(2022, 03, 22).midnight.utc, + Date.new(2022, 04, 22).midnight.utc + ].freeze + + scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } + scope :traces, -> { where(file_type: 3) } + end + + def up + return unless Gitlab.com? + + queue_background_migration_jobs_by_range_at_intervals( + JobArtifact.traces.in_targeted_timestamps, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20220131000000 b/db/schema_migrations/20220131000000 new file mode 100644 index 00000000000..f80ceb97f1c --- /dev/null +++ b/db/schema_migrations/20220131000000 @@ -0,0 +1 @@ +08326048e15f368f09bc10ebf5bee3e77e8b43813f66c19d24731497ca6a8485 \ No newline at end of file diff --git a/db/schema_migrations/20220131000001 b/db/schema_migrations/20220131000001 new file mode 100644 index 00000000000..72dce62bbce --- /dev/null +++ b/db/schema_migrations/20220131000001 @@ -0,0 +1 @@ +59fe701bcaa102b7e0c1496198fa4aeea6b2e59132c951d1c9d54562c5e3900e \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 357e332ee39..29a6d140d1f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -29522,6 +29522,8 @@ CREATE INDEX tmp_gitlab_subscriptions_max_seats_used_migration_2 ON gitlab_subsc CREATE INDEX tmp_idx_vulnerability_occurrences_on_id_where_report_type_7_99 ON vulnerability_occurrences USING btree (id) WHERE (report_type = ANY (ARRAY[7, 99])); +CREATE INDEX tmp_index_ci_job_artifacts_on_id_where_trace_and_expire_at ON ci_job_artifacts USING btree (id) WHERE ((file_type = 3) AND (expire_at = ANY (ARRAY['2021-04-22 00:00:00+00'::timestamp with time zone, '2021-05-22 00:00:00+00'::timestamp with time zone, '2021-06-22 00:00:00+00'::timestamp with time zone, '2022-01-22 00:00:00+00'::timestamp with time zone, '2022-02-22 00:00:00+00'::timestamp with time zone, '2022-03-22 00:00:00+00'::timestamp with time zone, '2022-04-22 00:00:00+00'::timestamp with time zone]))); + CREATE INDEX tmp_index_container_repositories_on_id_migration_state ON container_repositories USING btree (id, migration_state); CREATE INDEX tmp_index_for_namespace_id_migration_on_group_members ON members USING btree (id) WHERE ((member_namespace_id IS NULL) AND ((type)::text = 'GroupMember'::text)); diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index b66e88a8d60..913437c2d77 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -14,7 +14,7 @@ having an issue with GitLab, you may want to check your [support options](https: first, before attempting to use this information. WARNING: -If you administer GitLab you are expected to know these commands for your distribution +It is [beyond the scope of GitLab Support to assist in systems administration](https://about.gitlab.com/support/statement-of-support.html#training). GitLab administrators are expected to know these commands for their distribution of choice. If you are a GitLab Support Engineer, consider this a cross-reference to translate `yum` -> `apt-get` and the like. diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index cfa1fdba699..ceaa052008d 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -65,3 +65,12 @@ If the app install failed, you might need to delete `jira_connect_installations` 1. Open the [database console](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#access-postgresql). 1. Run `TRUNCATE TABLE jira_connect_installations CASCADE;`. + +#### Not authorized to access the file + +If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK's port public by following these steps: + +1. Open your GitLab workspace in Gitpod. +1. When the GDK is running, select **Ports** in the bottom-right corner. +1. On the left sidebar, select the port the GDK is listening to (typically `3000`). +1. If the port is marked as private, select the lock icon to make it public. diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index 9d8b98edba1..133ebda4360 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -771,7 +771,8 @@ Payload example: Merge request events are triggered when: - A new merge request is created. -- An existing merge request is updated, approved, unapproved, merged, or closed. +- An existing merge request is updated, approved (by all required approvers), unapproved, merged, or closed. +- An individual user adds or removes their approval to an existing merge request. - A commit is added in the source branch. - All threads are resolved on the merge request. @@ -783,6 +784,8 @@ The available values for `object_attributes.action` in the payload are: - `update` - `approved` - `unapproved` +- `approval` +- `unapproval` - `merge` Request header: diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb new file mode 100644 index 00000000000..d47aa76f24b --- /dev/null +++ b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removing expire_at timestamps that shouldn't have + # been written to traces on gitlab.com. + class RemoveAllTraceExpirationDates + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + # Stubbed class to connect to the CI database + # connects_to has to be called in abstract classes. + class MultiDbAdaptableClass < ActiveRecord::Base + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end + + # Stubbed class to access the ci_job_artifacts table + class JobArtifact < MultiDbAdaptableClass + include EachBatch + + self.table_name = 'ci_job_artifacts' + + TARGET_TIMESTAMPS = [ + Date.new(2021, 04, 22).midnight.utc, + Date.new(2021, 05, 22).midnight.utc, + Date.new(2021, 06, 22).midnight.utc, + Date.new(2022, 01, 22).midnight.utc, + Date.new(2022, 02, 22).midnight.utc, + Date.new(2022, 03, 22).midnight.utc, + Date.new(2022, 04, 22).midnight.utc + ].freeze + + scope :traces, -> { where(file_type: 3) } + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } + end + + def perform(start_id, end_id) + return unless Gitlab.com? + + JobArtifact.traces + .between(start_id, end_id) + .in_targeted_timestamps + .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index f867dbd4d68..5605d48c543 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,27 +4,13 @@ module Gitlab module CycleAnalytics module Summary class Base + include Gitlab::CycleAnalytics::Summary::Defaults + def initialize(project:, options:) @project = project @options = options end - def identifier - self.class.name.demodulize.underscore.to_sym - end - - def title - raise NotImplementedError, "Expected #{self.name} to implement title" - end - - def value - raise NotImplementedError, "Expected #{self.name} to implement value" - end - - def links - [] - end - private attr_reader :project, :options diff --git a/lib/gitlab/cycle_analytics/summary/defaults.rb b/lib/gitlab/cycle_analytics/summary/defaults.rb new file mode 100644 index 00000000000..468494a8ab8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/defaults.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Defaults + def identifier + self.class.name.demodulize.underscore.to_sym + end + + # :nocov: the class including this concern is expected to test this method. + def title + raise NotImplementedError, "Expected #{self.name} to implement title" + end + # :nocov: + + # :nocov: the class including this concern is expected to test this method. + def value + raise NotImplementedError, "Expected #{self.name} to implement value" + end + # :nocov: + + def links + [] + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb b/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb new file mode 100644 index 00000000000..8cdcec9621c --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_all_trace_expiration_dates_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveAllTraceExpirationDates, :migration, schema: 20220131000001 do + subject(:perform) { migration.perform(1, 99) } + + let(:migration) { described_class.new } + + let(:trace_in_range) { create_trace!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + let(:trace_outside_range) { create_trace!(id: 40, created_at: Date.new(2020, 06, 22), expire_at: Date.new(2021, 01, 22)) } + let(:trace_without_expiry) { create_trace!(id: 30, created_at: Date.new(2020, 06, 21), expire_at: nil) } + let(:archive_in_range) { create_archive!(id: 10, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + let(:trace_outside_id_range) { create_trace!(id: 100, created_at: Date.new(2020, 06, 20), expire_at: Date.new(2021, 01, 22)) } + + before do + table(:namespaces).create!(id: 1, name: 'the-namespace', path: 'the-path') + table(:projects).create!(id: 1, name: 'the-project', namespace_id: 1) + table(:ci_builds).create!(id: 1, allow_failure: false) + end + + context 'for self-hosted instances' do + it 'sets expire_at for artifacts in range to nil' do + expect { perform }.not_to change { trace_in_range.reload.expire_at } + end + + it 'does not change expire_at timestamps that are not set to midnight' do + expect { perform }.not_to change { trace_outside_range.reload.expire_at } + end + + it 'does not change expire_at timestamps that are set to midnight on a day other than the 22nd' do + expect { perform }.not_to change { trace_without_expiry.reload.expire_at } + end + + it 'does not touch artifacts outside id range' do + expect { perform }.not_to change { archive_in_range.reload.expire_at } + end + + it 'does not touch artifacts outside date range' do + expect { perform }.not_to change { trace_outside_id_range.reload.expire_at } + end + end + + private + + def create_trace!(**args) + table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 3) + end + + def create_archive!(**args) + table(:ci_job_artifacts).create!(**args, project_id: 1, job_id: 1, file_type: 1) + end +end