Add latest changes from gitlab-org/gitlab@master

GITALY_SERVER_VERSION

@@ -1 +1 @@
-10b19a9fe0fe008247056f4a90fde9006b8a7fbb
+71d527f4f16c1f0e76793f055def0299b375cc7d

app/assets/javascripts/ide/components/pipelines/list.vue

@@ -1,11 +1,16 @@
 <script>
 import { mapActions, mapGetters, mapState } from 'vuex';
 import { escape } from 'lodash';
-import { GlLoadingIcon, GlIcon, GlSafeHtmlDirective as SafeHtml } from '@gitlab/ui';
+import {
+  GlLoadingIcon,
+  GlIcon,
+  GlSafeHtmlDirective as SafeHtml,
+  GlTabs,
+  GlTab,
+  GlBadge,
+} from '@gitlab/ui';
 import { sprintf, __ } from '../../../locale';
 import CiIcon from '../../../vue_shared/components/ci_icon.vue';
-import Tabs from '../../../vue_shared/components/tabs/tabs';
-import Tab from '../../../vue_shared/components/tabs/tab.vue';
 import EmptyState from '../../../pipelines/components/pipelines_list/empty_state.vue';
 import JobsList from '../jobs/list.vue';
 
@@ -15,11 +20,12 @@ export default {
   components: {
     GlIcon,
     CiIcon,
-    Tabs,
-    Tab,
     JobsList,
     EmptyState,
     GlLoadingIcon,
+    GlTabs,
+    GlTab,
+    GlBadge,
   },
   directives: {
     SafeHtml,
@@ -88,22 +94,26 @@ export default {
         <p class="gl-mb-0 break-word">{{ latestPipeline.yamlError }}</p>
         <p v-safe-html="ciLintText" class="gl-mb-0"></p>
       </div>
-      <tabs v-else class="ide-pipeline-list">
+      <gl-tabs v-else>
-        <tab :active="!pipelineFailed">
+        <gl-tab :active="!pipelineFailed">
           <template #title>
             {{ __('Jobs') }}
-            <span v-if="jobsCount" class="badge badge-pill"> {{ jobsCount }} </span>
+            <gl-badge v-if="jobsCount" size="sm" class="gl-tab-counter-badge">{{
+              jobsCount
+            }}</gl-badge>
           </template>
           <jobs-list :loading="isLoadingJobs" :stages="stages" />
-        </tab>
+        </gl-tab>
-        <tab :active="pipelineFailed">
+        <gl-tab :active="pipelineFailed">
           <template #title>
             {{ __('Failed Jobs') }}
-            <span v-if="failedJobsCount" class="badge badge-pill"> {{ failedJobsCount }} </span>
+            <gl-badge v-if="failedJobsCount" size="sm" class="gl-tab-counter-badge">{{
+              failedJobsCount
+            }}</gl-badge>
           </template>
           <jobs-list :loading="isLoadingJobs" :stages="failedStages" />
-        </tab>
+        </gl-tab>
-      </tabs>
+      </gl-tabs>
     </template>
   </div>
 </template>

app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss

@@ -5,7 +5,7 @@
   $bs-input-focus-border: #80bdff;
   $bs-input-focus-box-shadow: rgba(0, 123, 255, 0.25);
 
-  a:not(.btn),
+  a:not(.btn):not(.gl-tab-nav-item),
   .gl-button.btn-link,
   .gl-button.btn-link:hover,
   .gl-button.btn-link:focus,

app/assets/stylesheets/page_bundles/ide.scss

@@ -915,12 +915,6 @@ $ide-commit-header-height: 48px;
   }
 }
 
-.ide-pipeline-list {
-  flex: 1;
-  overflow: auto;
-  padding: 0 $gl-padding;
-}
-
 .ide-pipeline-header {
   min-height: 55px;
   padding-left: $gl-padding;

app/models/ci/job_artifact.rb

@@ -169,6 +169,7 @@ module Ci
 
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
     scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked).order(expire_at: :desc) }
+    scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
 
     scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }
 

app/services/ci/destroy_expired_job_artifacts_service.rb

@@ -6,45 +6,106 @@ module Ci
     include ::Gitlab::LoopHelpers
 
     BATCH_SIZE = 100
-    LOOP_TIMEOUT = 45.minutes
+    LOOP_TIMEOUT = 5.minutes
+    LEGACY_LOOP_TIMEOUT = 45.minutes
     LOOP_LIMIT = 1000
     EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
-    LOCK_TIMEOUT = 50.minutes
+    LOCK_TIMEOUT = 10.minutes
+    LEGACY_LOCK_TIMEOUT = 50.minutes
 
     ##
     # Destroy expired job artifacts on GitLab instance
     #
-    # This destroy process cannot run for more than 45 minutes. This is for
+    # This destroy process cannot run for more than 10 minutes. This is for
     # preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently,
     # which is scheduled at every hour.
     def execute
-      in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
+      in_lock(EXCLUSIVE_LOCK_KEY, ttl: lock_timeout, retries: 1) do
-        loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
+        loop_until(timeout: loop_timeout, limit: LOOP_LIMIT) do
-          destroy_batch(Ci::JobArtifact) || destroy_batch(Ci::PipelineArtifact)
+          destroy_artifacts_batch
         end
       end
     end
 
     private
 
-    def destroy_batch(klass)
+    def destroy_artifacts_batch
-      artifact_batch = if klass == Ci::JobArtifact
+      destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch
-                         klass.expired(BATCH_SIZE).unlocked
+    end
-                       else
-                         klass.expired(BATCH_SIZE)
-                       end
 
-      artifacts = artifact_batch.to_a
+    def destroy_job_artifacts_batch
+      artifacts = Ci::JobArtifact
+        .expired(BATCH_SIZE)
+        .unlocked
+        .with_destroy_preloads
+        .to_a
 
       return false if artifacts.empty?
 
-      artifacts.each(&:destroy!)
+      if parallel_destroy?
-      run_after_destroy(artifacts)
+        parallel_destroy_batch(artifacts)
+      else
+        legacy_destroy_batch(artifacts)
+        destroy_related_records_for(artifacts)
+      end
 
-      true # This is required because of the design of `loop_until` method.
+      true
     end
 
-    def run_after_destroy(artifacts); end
+    def destroy_pipeline_artifacts_batch
+      artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
+      return false if artifacts.empty?
+
+      legacy_destroy_batch(artifacts)
+      true
+    end
+
+    def parallel_destroy?
+      ::Feature.enabled?(:ci_delete_objects)
+    end
+
+    def legacy_destroy_batch(artifacts)
+      artifacts.each(&:destroy!)
+    end
+
+    def parallel_destroy_batch(job_artifacts)
+      Ci::DeletedObject.transaction do
+        Ci::DeletedObject.bulk_import(job_artifacts)
+        Ci::JobArtifact.id_in(job_artifacts.map(&:id)).delete_all
+        destroy_related_records_for(job_artifacts)
+      end
+
+      # This is executed outside of the transaction because it depends on Redis
+      update_statistics_for(job_artifacts)
+    end
+
+    # This method is implemented in EE and it must do only database work
+    def destroy_related_records_for(job_artifacts); end
+
+    def update_statistics_for(job_artifacts)
+      artifacts_by_project = job_artifacts.group_by(&:project)
+      artifacts_by_project.each do |project, artifacts|
+        delta = -artifacts.sum { |artifact| artifact.size.to_i }
+        ProjectStatistics.increment_statistic(
+          project, Ci::JobArtifact.project_statistics_name, delta)
+      end
+    end
+
+    def loop_timeout
+      if parallel_destroy?
+        LOOP_TIMEOUT
+      else
+        LEGACY_LOOP_TIMEOUT
+      end
+    end
+
+    def lock_timeout
+      if parallel_destroy?
+        LOCK_TIMEOUT
+      else
+        LEGACY_LOCK_TIMEOUT
+      end
+    end
   end
 end
 

changelogs/unreleased/230724-matej-tabs-migration-ide-pipelines-list.yml

@@ -0,0 +1,5 @@
+---
+title: Update Web IDE pipelines panel to use our design system component
+merge_request: 45007
+author: matejlatin
+type: other

config/feature_flags/development/ci_delete_objects.yml

@@ -0,0 +1,7 @@
+---
+name: ci_delete_objects
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42242
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/247103
+group: group::continuous integration
+type: development
+default_enabled: false

doc/user/project/settings/import_export.md

@@ -1,6 +1,6 @@
 ---
-stage: Create
+stage: Manage
-group: Source Code
+group: Import
 info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers"
 type: reference, howto
 ---

spec/frontend/ide/components/pipelines/list_spec.js

@@ -1,11 +1,10 @@
 import { shallowMount, createLocalVue } from '@vue/test-utils';
 import Vuex from 'vuex';
-import { GlLoadingIcon } from '@gitlab/ui';
+import { GlLoadingIcon, GlTab } from '@gitlab/ui';
 import { TEST_HOST } from 'helpers/test_constants';
 import { pipelines } from 'jest/ide/mock_data';
 import List from '~/ide/components/pipelines/list.vue';
 import JobsList from '~/ide/components/jobs/list.vue';
-import Tab from '~/vue_shared/components/tabs/tab.vue';
 import CiIcon from '~/vue_shared/components/ci_icon.vue';
 import IDEServices from '~/ide/services';
 
@@ -167,7 +166,7 @@ describe('IDE pipelines list', () => {
         createComponent({}, { ...withLatestPipelineState, stages, isLoadingJobs });
 
         const jobProps = wrapper
-          .findAll(Tab)
+          .findAll(GlTab)
           .at(0)
           .find(JobsList)
           .props();
@@ -182,7 +181,7 @@ describe('IDE pipelines list', () => {
         createComponent({}, { ...withLatestPipelineState, isLoadingJobs });
 
         const jobProps = wrapper
-          .findAll(Tab)
+          .findAll(GlTab)
           .at(1)
           .find(JobsList)
           .props();

spec/services/ci/destroy_expired_job_artifacts_service_spec.rb

@@ -5,26 +5,77 @@ require 'spec_helper'
 RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared_state do
   include ExclusiveLeaseHelpers
 
+  let(:service) { described_class.new }
+
   describe '.execute' do
     subject { service.execute }
 
-    let(:service) { described_class.new }
+    let_it_be(:artifact, reload: true) do
-
+      create(:ci_job_artifact, expire_at: 1.day.ago)
-    let_it_be(:artifact) { create(:ci_job_artifact, expire_at: 1.day.ago) }
+    end
 
     before(:all) do
       artifact.job.pipeline.unlocked!
     end
 
     context 'when artifact is expired' do
-      context 'when artifact is not locked' do
+      context 'with preloaded relationships' do
         before do
-          artifact.job.pipeline.unlocked!
+          job = create(:ci_build, pipeline: artifact.job.pipeline)
+          create(:ci_job_artifact, :archive, :expired, job: job)
+
+          stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 1)
         end
 
-        it 'destroys job artifact' do
+        it 'performs the smallest number of queries for job_artifacts' do
+          log = ActiveRecord::QueryRecorder.new { subject }
+
+          # SELECT expired ci_job_artifacts
+          # PRELOAD projects, routes, project_statistics
+          # BEGIN
+          # INSERT into ci_deleted_objects
+          # DELETE loaded ci_job_artifacts
+          # DELETE security_findings  -- for EE
+          # COMMIT
+          expect(log.count).to be_within(1).of(8)
+        end
+      end
+
+      context 'when artifact is not locked' do
+        it 'deletes job artifact record' do
           expect { subject }.to change { Ci::JobArtifact.count }.by(-1)
         end
+
+        context 'when the artifact does not a file attached to it' do
+          it 'does not create deleted objects' do
+            expect(artifact.exists?).to be_falsy # sanity check
+
+            expect { subject }.not_to change { Ci::DeletedObject.count }
+          end
+        end
+
+        context 'when the artifact has a file attached to it' do
+          before do
+            artifact.file = fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip')
+            artifact.save!
+          end
+
+          it 'creates a deleted object' do
+            expect { subject }.to change { Ci::DeletedObject.count }.by(1)
+          end
+
+          it 'resets project statistics' do
+            expect(ProjectStatistics).to receive(:increment_statistic).once
+              .with(artifact.project, :build_artifacts_size, -artifact.file.size)
+              .and_call_original
+
+            subject
+          end
+
+          it 'does not remove the files' do
+            expect { subject }.not_to change { artifact.file.exists? }
+          end
+        end
       end
 
       context 'when artifact is locked' do
@@ -61,14 +112,52 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
     context 'when failed to destroy artifact' do
       before do
         stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
-
-        allow_any_instance_of(Ci::JobArtifact)
-          .to receive(:destroy!)
-          .and_raise(ActiveRecord::RecordNotDestroyed)
       end
 
-      it 'raises an exception and stop destroying' do
+      context 'with ci_delete_objects disabled' do
-        expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
+        before do
+          stub_feature_flags(ci_delete_objects: false)
+
+          allow_any_instance_of(Ci::JobArtifact)
+            .to receive(:destroy!)
+            .and_raise(ActiveRecord::RecordNotDestroyed)
+        end
+
+        it 'raises an exception and stop destroying' do
+          expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
+        end
+      end
+
+      context 'with ci_delete_objects enabled' do
+        context 'when the import fails' do
+          before do
+            stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
+            expect(Ci::DeletedObject)
+              .to receive(:bulk_import)
+              .once
+              .and_raise(ActiveRecord::RecordNotDestroyed)
+          end
+
+          it 'raises an exception and stop destroying' do
+            expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
+                              .and not_change { Ci::JobArtifact.count }.from(1)
+          end
+        end
+
+        context 'when the delete fails' do
+          before do
+            stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
+            expect(Ci::JobArtifact)
+              .to receive(:id_in)
+              .once
+              .and_raise(ActiveRecord::RecordNotDestroyed)
+          end
+
+          it 'raises an exception rolls back the insert' do
+            expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
+                              .and not_change { Ci::DeletedObject.count }.from(0)
+          end
+        end
       end
     end
 
@@ -85,7 +174,7 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
     context 'when timeout happens' do
       before do
         stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_TIMEOUT', 1.second)
-        allow_any_instance_of(described_class).to receive(:destroy_batch) { true }
+        allow_any_instance_of(described_class).to receive(:destroy_artifacts_batch) { true }
       end
 
       it 'returns false and does not continue destroying' do
@@ -176,4 +265,16 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
       end
     end
   end
+
+  describe '.destroy_job_artifacts_batch' do
+    it 'returns a falsy value without artifacts' do
+      expect(service.send(:destroy_job_artifacts_batch)).to be_falsy
+    end
+  end
+
+  describe '.destroy_pipeline_artifacts_batch' do
+    it 'returns a falsy value without artifacts' do
+      expect(service.send(:destroy_pipeline_artifacts_batch)).to be_falsy
+    end
+  end
 end