Merge branch '43949-verify-job-artifacts' into 'master'
Resolve "Foreground verification of job artifacts" Closes #43949 See merge request gitlab-org/gitlab-ce!17578
This commit is contained in:
commit
dad547a5f1
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
title: Implement foreground verification of CI artifacts
|
||||||
|
merge_request: 17578
|
||||||
|
author:
|
||||||
|
type: added
|
|
@ -84,12 +84,14 @@ checks using those checksums can be run. These checks also detect missing files.
|
||||||
|
|
||||||
Currently, integrity checks are supported for the following types of file:
|
Currently, integrity checks are supported for the following types of file:
|
||||||
|
|
||||||
|
* CI artifacts
|
||||||
* LFS objects
|
* LFS objects
|
||||||
* User uploads
|
* User uploads
|
||||||
|
|
||||||
**Omnibus Installation**
|
**Omnibus Installation**
|
||||||
|
|
||||||
```
|
```
|
||||||
|
sudo gitlab-rake gitlab:artifacts:check
|
||||||
sudo gitlab-rake gitlab:lfs:check
|
sudo gitlab-rake gitlab:lfs:check
|
||||||
sudo gitlab-rake gitlab:uploads:check
|
sudo gitlab-rake gitlab:uploads:check
|
||||||
```
|
```
|
||||||
|
@ -97,6 +99,7 @@ sudo gitlab-rake gitlab:uploads:check
|
||||||
**Source Installation**
|
**Source Installation**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
sudo -u git -H bundle exec rake gitlab:artifacts:check RAILS_ENV=production
|
||||||
sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production
|
sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production
|
||||||
sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
|
sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
|
||||||
```
|
```
|
||||||
|
@ -112,6 +115,7 @@ Variable | Type | Description
|
||||||
`VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized.
|
`VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
sudo gitlab-rake gitlab:artifacts:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||||
sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250
|
sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||||
sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
|
sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
|
||||||
```
|
```
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
module Gitlab
|
||||||
|
module Verify
|
||||||
|
class JobArtifacts < BatchVerifier
|
||||||
|
def name
|
||||||
|
'Job artifacts'
|
||||||
|
end
|
||||||
|
|
||||||
|
def describe(object)
|
||||||
|
"Job artifact: #{object.id}"
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def relation
|
||||||
|
::Ci::JobArtifact.all
|
||||||
|
end
|
||||||
|
|
||||||
|
def expected_checksum(artifact)
|
||||||
|
artifact.file_sha256
|
||||||
|
end
|
||||||
|
|
||||||
|
def actual_checksum(artifact)
|
||||||
|
Digest::SHA256.file(artifact.file.path).hexdigest
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,8 @@
|
||||||
|
namespace :gitlab do
|
||||||
|
namespace :artifacts do
|
||||||
|
desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts'
|
||||||
|
task check: :environment do
|
||||||
|
Gitlab::Verify::RakeTask.run!(Gitlab::Verify::JobArtifacts)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -35,5 +35,11 @@ FactoryBot.define do
|
||||||
Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
|
Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
trait :correct_checksum do
|
||||||
|
after(:build) do |artifact, evaluator|
|
||||||
|
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
require 'spec_helper'
|
||||||
|
|
||||||
|
describe Gitlab::Verify::JobArtifacts do
|
||||||
|
include GitlabVerifyHelpers
|
||||||
|
|
||||||
|
it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
|
||||||
|
let!(:objects) { create_list(:ci_job_artifact, 3, :archive) }
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#run_batches' do
|
||||||
|
let(:failures) { collect_failures }
|
||||||
|
let(:failure) { failures[artifact] }
|
||||||
|
|
||||||
|
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
|
||||||
|
|
||||||
|
it 'passes artifacts with the correct file' do
|
||||||
|
expect(failures).to eq({})
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails artifacts with a missing file' do
|
||||||
|
FileUtils.rm_f(artifact.file.path)
|
||||||
|
|
||||||
|
expect(failures.keys).to contain_exactly(artifact)
|
||||||
|
expect(failure).to be_a(Errno::ENOENT)
|
||||||
|
expect(failure.to_s).to include(artifact.file.path)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails artifacts with a mismatched checksum' do
|
||||||
|
File.truncate(artifact.file.path, 0)
|
||||||
|
|
||||||
|
expect(failures.keys).to contain_exactly(artifact)
|
||||||
|
expect(failure.to_s).to include('Checksum mismatch')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,34 @@
|
||||||
|
require 'rake_helper'
|
||||||
|
|
||||||
|
describe 'gitlab:artifacts rake tasks' do
|
||||||
|
describe 'check' do
|
||||||
|
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
Rake.application.rake_require('tasks/gitlab/artifacts/check')
|
||||||
|
stub_env('VERBOSE' => 'true')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'outputs the integrity check for each batch' do
|
||||||
|
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Failures: 0/).to_stdout
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'errors out about missing files on the file system' do
|
||||||
|
FileUtils.rm_f(artifact.file.path)
|
||||||
|
|
||||||
|
expect { run_rake_task('gitlab:artifacts:check') }.to output(/No such file.*#{Regexp.quote(artifact.file.path)}/).to_stdout
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'errors out about invalid checksum' do
|
||||||
|
artifact.update_column(:file_sha256, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
|
||||||
|
|
||||||
|
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum mismatch/).to_stdout
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'errors out about missing checksum' do
|
||||||
|
artifact.update_column(:file_sha256, nil)
|
||||||
|
|
||||||
|
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum missing/).to_stdout
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue