Merge branch '43949-verify-job-artifacts' into 'master'
Resolve "Foreground verification of job artifacts" Closes #43949 See merge request gitlab-org/gitlab-ce!17578
This commit is contained in:
commit
dad547a5f1
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
title: Implement foreground verification of CI artifacts
|
||||
merge_request: 17578
|
||||
author:
|
||||
type: added
|
|
@ -84,12 +84,14 @@ checks using those checksums can be run. These checks also detect missing files.
|
|||
|
||||
Currently, integrity checks are supported for the following types of file:
|
||||
|
||||
* CI artifacts
|
||||
* LFS objects
|
||||
* User uploads
|
||||
|
||||
**Omnibus Installation**
|
||||
|
||||
```
|
||||
sudo gitlab-rake gitlab:artifacts:check
|
||||
sudo gitlab-rake gitlab:lfs:check
|
||||
sudo gitlab-rake gitlab:uploads:check
|
||||
```
|
||||
|
@ -97,6 +99,7 @@ sudo gitlab-rake gitlab:uploads:check
|
|||
**Source Installation**
|
||||
|
||||
```bash
|
||||
sudo -u git -H bundle exec rake gitlab:artifacts:check RAILS_ENV=production
|
||||
sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production
|
||||
sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
|
||||
```
|
||||
|
@ -112,6 +115,7 @@ Variable | Type | Description
|
|||
`VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized.
|
||||
|
||||
```bash
|
||||
sudo gitlab-rake gitlab:artifacts:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||
sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||
sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||
```
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
module Gitlab
|
||||
module Verify
|
||||
class JobArtifacts < BatchVerifier
|
||||
def name
|
||||
'Job artifacts'
|
||||
end
|
||||
|
||||
def describe(object)
|
||||
"Job artifact: #{object.id}"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def relation
|
||||
::Ci::JobArtifact.all
|
||||
end
|
||||
|
||||
def expected_checksum(artifact)
|
||||
artifact.file_sha256
|
||||
end
|
||||
|
||||
def actual_checksum(artifact)
|
||||
Digest::SHA256.file(artifact.file.path).hexdigest
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,8 @@
|
|||
namespace :gitlab do
|
||||
namespace :artifacts do
|
||||
desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts'
|
||||
task check: :environment do
|
||||
Gitlab::Verify::RakeTask.run!(Gitlab::Verify::JobArtifacts)
|
||||
end
|
||||
end
|
||||
end
|
|
@ -35,5 +35,11 @@ FactoryBot.define do
|
|||
Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
|
||||
end
|
||||
end
|
||||
|
||||
trait :correct_checksum do
|
||||
after(:build) do |artifact, evaluator|
|
||||
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Gitlab::Verify::JobArtifacts do
|
||||
include GitlabVerifyHelpers
|
||||
|
||||
it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
|
||||
let!(:objects) { create_list(:ci_job_artifact, 3, :archive) }
|
||||
end
|
||||
|
||||
describe '#run_batches' do
|
||||
let(:failures) { collect_failures }
|
||||
let(:failure) { failures[artifact] }
|
||||
|
||||
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
|
||||
|
||||
it 'passes artifacts with the correct file' do
|
||||
expect(failures).to eq({})
|
||||
end
|
||||
|
||||
it 'fails artifacts with a missing file' do
|
||||
FileUtils.rm_f(artifact.file.path)
|
||||
|
||||
expect(failures.keys).to contain_exactly(artifact)
|
||||
expect(failure).to be_a(Errno::ENOENT)
|
||||
expect(failure.to_s).to include(artifact.file.path)
|
||||
end
|
||||
|
||||
it 'fails artifacts with a mismatched checksum' do
|
||||
File.truncate(artifact.file.path, 0)
|
||||
|
||||
expect(failures.keys).to contain_exactly(artifact)
|
||||
expect(failure.to_s).to include('Checksum mismatch')
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,34 @@
|
|||
require 'rake_helper'
|
||||
|
||||
describe 'gitlab:artifacts rake tasks' do
|
||||
describe 'check' do
|
||||
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
|
||||
|
||||
before do
|
||||
Rake.application.rake_require('tasks/gitlab/artifacts/check')
|
||||
stub_env('VERBOSE' => 'true')
|
||||
end
|
||||
|
||||
it 'outputs the integrity check for each batch' do
|
||||
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Failures: 0/).to_stdout
|
||||
end
|
||||
|
||||
it 'errors out about missing files on the file system' do
|
||||
FileUtils.rm_f(artifact.file.path)
|
||||
|
||||
expect { run_rake_task('gitlab:artifacts:check') }.to output(/No such file.*#{Regexp.quote(artifact.file.path)}/).to_stdout
|
||||
end
|
||||
|
||||
it 'errors out about invalid checksum' do
|
||||
artifact.update_column(:file_sha256, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
|
||||
|
||||
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum mismatch/).to_stdout
|
||||
end
|
||||
|
||||
it 'errors out about missing checksum' do
|
||||
artifact.update_column(:file_sha256, nil)
|
||||
|
||||
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum missing/).to_stdout
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue