kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2020-10-23 12:08:57 +00:00
parent ca7bdd871e
commit b17c74a7e2
11 changed files with 235 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
GITALY_SERVER_VERSION
View File

@ -1 +1 @@
10b19a9fe0fe008247056f4a90fde9006b8a7fbb
71d527f4f16c1f0e76793f055def0299b375cc7d

36
app/assets/javascripts/ide/components/pipelines/list.vue
View File

@ -1,11 +1,16 @@
<script>
import { mapActions, mapGetters, mapState } from 'vuex';
import { escape } from 'lodash';
import { GlLoadingIcon, GlIcon, GlSafeHtmlDirective as SafeHtml } from '@gitlab/ui';
import {
GlLoadingIcon,
GlIcon,
GlSafeHtmlDirective as SafeHtml,
GlTabs,
GlTab,
GlBadge,
} from '@gitlab/ui';
import { sprintf, __ } from '../../../locale';
import CiIcon from '../../../vue_shared/components/ci_icon.vue';
import Tabs from '../../../vue_shared/components/tabs/tabs';
import Tab from '../../../vue_shared/components/tabs/tab.vue';
import EmptyState from '../../../pipelines/components/pipelines_list/empty_state.vue';
import JobsList from '../jobs/list.vue';
@ -15,11 +20,12 @@ export default {
components: {
GlIcon,
CiIcon,
Tabs,
Tab,
JobsList,
EmptyState,
GlLoadingIcon,
GlTabs,
GlTab,
GlBadge,
},
directives: {
SafeHtml,
@ -88,22 +94,26 @@ export default {
<p class="gl-mb-0 break-word">{{ latestPipeline.yamlError }}</p>
<p v-safe-html="ciLintText" class="gl-mb-0"></p>
</div>
<tabs v-else class="ide-pipeline-list">
<tab :active="!pipelineFailed">
<gl-tabs v-else>
<gl-tab :active="!pipelineFailed">
<template #title>
{{ __('Jobs') }}
<span v-if="jobsCount" class="badge badge-pill"> {{ jobsCount }} </span>
<gl-badge v-if="jobsCount" size="sm" class="gl-tab-counter-badge">{{
jobsCount
}}</gl-badge>
</template>
<jobs-list :loading="isLoadingJobs" :stages="stages" />
</tab>
<tab :active="pipelineFailed">
</gl-tab>
<gl-tab :active="pipelineFailed">
<template #title>
{{ __('Failed Jobs') }}
<span v-if="failedJobsCount" class="badge badge-pill"> {{ failedJobsCount }} </span>
<gl-badge v-if="failedJobsCount" size="sm" class="gl-tab-counter-badge">{{
failedJobsCount
}}</gl-badge>
</template>
<jobs-list :loading="isLoadingJobs" :stages="failedStages" />
</tab>
</tabs>
</gl-tab>
</gl-tabs>
</template>
</div>
</template>

2
app/assets/stylesheets/page_bundles/_ide_theme_overrides.scss
View File

@ -5,7 +5,7 @@
$bs-input-focus-border: #80bdff;
$bs-input-focus-box-shadow: rgba(0, 123, 255, 0.25);
a:not(.btn),
a:not(.btn):not(.gl-tab-nav-item),
.gl-button.btn-link,
.gl-button.btn-link:hover,
.gl-button.btn-link:focus,

6
app/assets/stylesheets/page_bundles/ide.scss
View File

@ -915,12 +915,6 @@ $ide-commit-header-height: 48px;
}
}
.ide-pipeline-list {
flex: 1;
overflow: auto;
padding: 0 $gl-padding;
}
.ide-pipeline-header {
min-height: 55px;
padding-left: $gl-padding;

1
app/models/ci/job_artifact.rb
View File

@ -169,6 +169,7 @@ module Ci
scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked).order(expire_at: :desc) }
scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }

95
app/services/ci/destroy_expired_job_artifacts_service.rb
View File

@ -6,45 +6,106 @@ module Ci
include ::Gitlab::LoopHelpers
BATCH_SIZE = 100
LOOP_TIMEOUT = 45.minutes
LOOP_TIMEOUT = 5.minutes
LEGACY_LOOP_TIMEOUT = 45.minutes
LOOP_LIMIT = 1000
EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
LOCK_TIMEOUT = 50.minutes
LOCK_TIMEOUT = 10.minutes
LEGACY_LOCK_TIMEOUT = 50.minutes
##
# Destroy expired job artifacts on GitLab instance
#
# This destroy process cannot run for more than 45 minutes. This is for
# This destroy process cannot run for more than 10 minutes. This is for
# preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently,
# which is scheduled at every hour.
def execute
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
destroy_batch(Ci::JobArtifact) || destroy_batch(Ci::PipelineArtifact)
in_lock(EXCLUSIVE_LOCK_KEY, ttl: lock_timeout, retries: 1) do
loop_until(timeout: loop_timeout, limit: LOOP_LIMIT) do
destroy_artifacts_batch
end
end
end
private
def destroy_batch(klass)
artifact_batch = if klass == Ci::JobArtifact
klass.expired(BATCH_SIZE).unlocked
else
klass.expired(BATCH_SIZE)
end
def destroy_artifacts_batch
destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch
end
artifacts = artifact_batch.to_a
def destroy_job_artifacts_batch
artifacts = Ci::JobArtifact
.expired(BATCH_SIZE)
.unlocked
.with_destroy_preloads
.to_a
return false if artifacts.empty?
artifacts.each(&:destroy!)
run_after_destroy(artifacts)
if parallel_destroy?
parallel_destroy_batch(artifacts)
else
legacy_destroy_batch(artifacts)
destroy_related_records_for(artifacts)
end
true # This is required because of the design of `loop_until` method.
true
end
def run_after_destroy(artifacts); end
def destroy_pipeline_artifacts_batch
artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
return false if artifacts.empty?
legacy_destroy_batch(artifacts)
true
end
def parallel_destroy?
::Feature.enabled?(:ci_delete_objects)
end
def legacy_destroy_batch(artifacts)
artifacts.each(&:destroy!)
end
def parallel_destroy_batch(job_artifacts)
Ci::DeletedObject.transaction do
Ci::DeletedObject.bulk_import(job_artifacts)
Ci::JobArtifact.id_in(job_artifacts.map(&:id)).delete_all
destroy_related_records_for(job_artifacts)
end
# This is executed outside of the transaction because it depends on Redis
update_statistics_for(job_artifacts)
end
# This method is implemented in EE and it must do only database work
def destroy_related_records_for(job_artifacts); end
def update_statistics_for(job_artifacts)
artifacts_by_project = job_artifacts.group_by(&:project)
artifacts_by_project.each do |project, artifacts|
delta = -artifacts.sum { |artifact| artifact.size.to_i }
ProjectStatistics.increment_statistic(
project, Ci::JobArtifact.project_statistics_name, delta)
end
end
def loop_timeout
if parallel_destroy?
LOOP_TIMEOUT
else
LEGACY_LOOP_TIMEOUT
end
end
def lock_timeout
if parallel_destroy?
LOCK_TIMEOUT
else
LEGACY_LOCK_TIMEOUT
end
end
end
end

5
changelogs/unreleased/230724-matej-tabs-migration-ide-pipelines-list.yml Normal file
View File

@ -0,0 +1,5 @@
---
title: Update Web IDE pipelines panel to use our design system component
merge_request: 45007
author: matejlatin
type: other

7
config/feature_flags/development/ci_delete_objects.yml Normal file
View File

@ -0,0 +1,7 @@
---
name: ci_delete_objects
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42242
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/247103
group: group::continuous integration
type: development
default_enabled: false

4
doc/user/project/settings/import_export.md
View File

@ -1,6 +1,6 @@
---
stage: Create
group: Source Code
stage: Manage
group: Import
info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers"
type: reference, howto
---

7
spec/frontend/ide/components/pipelines/list_spec.js
View File

@ -1,11 +1,10 @@
import { shallowMount, createLocalVue } from '@vue/test-utils';
import Vuex from 'vuex';
import { GlLoadingIcon } from '@gitlab/ui';
import { GlLoadingIcon, GlTab } from '@gitlab/ui';
import { TEST_HOST } from 'helpers/test_constants';
import { pipelines } from 'jest/ide/mock_data';
import List from '~/ide/components/pipelines/list.vue';
import JobsList from '~/ide/components/jobs/list.vue';
import Tab from '~/vue_shared/components/tabs/tab.vue';
import CiIcon from '~/vue_shared/components/ci_icon.vue';
import IDEServices from '~/ide/services';
@ -167,7 +166,7 @@ describe('IDE pipelines list', () => {
createComponent({}, { ...withLatestPipelineState, stages, isLoadingJobs });
const jobProps = wrapper
.findAll(Tab)
.findAll(GlTab)
.at(0)
.find(JobsList)
.props();
@ -182,7 +181,7 @@ describe('IDE pipelines list', () => {
createComponent({}, { ...withLatestPipelineState, isLoadingJobs });
const jobProps = wrapper
.findAll(Tab)
.findAll(GlTab)
.at(1)
.find(JobsList)
.props();

127
spec/services/ci/destroy_expired_job_artifacts_service_spec.rb
View File

@ -5,26 +5,77 @@ require 'spec_helper'
RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared_state do
include ExclusiveLeaseHelpers
let(:service) { described_class.new }
describe '.execute' do
subject { service.execute }
let(:service) { described_class.new }
let_it_be(:artifact) { create(:ci_job_artifact, expire_at: 1.day.ago) }
let_it_be(:artifact, reload: true) do
create(:ci_job_artifact, expire_at: 1.day.ago)
end
before(:all) do
artifact.job.pipeline.unlocked!
end
context 'when artifact is expired' do
context 'when artifact is not locked' do
context 'with preloaded relationships' do
before do
artifact.job.pipeline.unlocked!
job = create(:ci_build, pipeline: artifact.job.pipeline)
create(:ci_job_artifact, :archive, :expired, job: job)
stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 1)
end
it 'destroys job artifact' do
it 'performs the smallest number of queries for job_artifacts' do
log = ActiveRecord::QueryRecorder.new { subject }
# SELECT expired ci_job_artifacts
# PRELOAD projects, routes, project_statistics
# BEGIN
# INSERT into ci_deleted_objects
# DELETE loaded ci_job_artifacts
# DELETE security_findings -- for EE
# COMMIT
expect(log.count).to be_within(1).of(8)
end
end
context 'when artifact is not locked' do
it 'deletes job artifact record' do
expect { subject }.to change { Ci::JobArtifact.count }.by(-1)
end
context 'when the artifact does not a file attached to it' do
it 'does not create deleted objects' do
expect(artifact.exists?).to be_falsy # sanity check
expect { subject }.not_to change { Ci::DeletedObject.count }
end
end
context 'when the artifact has a file attached to it' do
before do
artifact.file = fixture_file_upload(Rails.root.join('spec/fixtures/ci_build_artifacts.zip'), 'application/zip')
artifact.save!
end
it 'creates a deleted object' do
expect { subject }.to change { Ci::DeletedObject.count }.by(1)
end
it 'resets project statistics' do
expect(ProjectStatistics).to receive(:increment_statistic).once
.with(artifact.project, :build_artifacts_size, -artifact.file.size)
.and_call_original
subject
end
it 'does not remove the files' do
expect { subject }.not_to change { artifact.file.exists? }
end
end
end
context 'when artifact is locked' do
@ -61,14 +112,52 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
context 'when failed to destroy artifact' do
before do
stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
allow_any_instance_of(Ci::JobArtifact)
.to receive(:destroy!)
.and_raise(ActiveRecord::RecordNotDestroyed)
end
it 'raises an exception and stop destroying' do
expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
context 'with ci_delete_objects disabled' do
before do
stub_feature_flags(ci_delete_objects: false)
allow_any_instance_of(Ci::JobArtifact)
.to receive(:destroy!)
.and_raise(ActiveRecord::RecordNotDestroyed)
end
it 'raises an exception and stop destroying' do
expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
end
end
context 'with ci_delete_objects enabled' do
context 'when the import fails' do
before do
stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
expect(Ci::DeletedObject)
.to receive(:bulk_import)
.once
.and_raise(ActiveRecord::RecordNotDestroyed)
end
it 'raises an exception and stop destroying' do
expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
.and not_change { Ci::JobArtifact.count }.from(1)
end
end
context 'when the delete fails' do
before do
stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_LIMIT', 10)
expect(Ci::JobArtifact)
.to receive(:id_in)
.once
.and_raise(ActiveRecord::RecordNotDestroyed)
end
it 'raises an exception rolls back the insert' do
expect { subject }.to raise_error(ActiveRecord::RecordNotDestroyed)
.and not_change { Ci::DeletedObject.count }.from(0)
end
end
end
end
@ -85,7 +174,7 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
context 'when timeout happens' do
before do
stub_const('Ci::DestroyExpiredJobArtifactsService::LOOP_TIMEOUT', 1.second)
allow_any_instance_of(described_class).to receive(:destroy_batch) { true }
allow_any_instance_of(described_class).to receive(:destroy_artifacts_batch) { true }
end
it 'returns false and does not continue destroying' do
@ -176,4 +265,16 @@ RSpec.describe Ci::DestroyExpiredJobArtifactsService, :clean_gitlab_redis_shared
end
end
end
describe '.destroy_job_artifacts_batch' do
it 'returns a falsy value without artifacts' do
expect(service.send(:destroy_job_artifacts_batch)).to be_falsy
end
end
describe '.destroy_pipeline_artifacts_batch' do
it 'returns a falsy value without artifacts' do
expect(service.send(:destroy_pipeline_artifacts_batch)).to be_falsy
end
end
end