Commit dad547a5 authored by Sean McGivern's avatar Sean McGivern

Merge branch '43949-verify-job-artifacts' into 'master'

Resolve "Foreground verification of job artifacts"

Closes #43949

See merge request gitlab-org/gitlab-ce!17578
parents f2723fc3 4419d7ea
---
title: Implement foreground verification of CI artifacts
merge_request: 17578
author:
type: added
...@@ -84,12 +84,14 @@ checks using those checksums can be run. These checks also detect missing files. ...@@ -84,12 +84,14 @@ checks using those checksums can be run. These checks also detect missing files.
Currently, integrity checks are supported for the following types of file: Currently, integrity checks are supported for the following types of file:
* CI artifacts
* LFS objects * LFS objects
* User uploads * User uploads
**Omnibus Installation** **Omnibus Installation**
``` ```
sudo gitlab-rake gitlab:artifacts:check
sudo gitlab-rake gitlab:lfs:check sudo gitlab-rake gitlab:lfs:check
sudo gitlab-rake gitlab:uploads:check sudo gitlab-rake gitlab:uploads:check
``` ```
...@@ -97,6 +99,7 @@ sudo gitlab-rake gitlab:uploads:check ...@@ -97,6 +99,7 @@ sudo gitlab-rake gitlab:uploads:check
**Source Installation** **Source Installation**
```bash ```bash
sudo -u git -H bundle exec rake gitlab:artifacts:check RAILS_ENV=production
sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production
sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
``` ```
...@@ -112,6 +115,7 @@ Variable | Type | Description ...@@ -112,6 +115,7 @@ Variable | Type | Description
`VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized. `VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized.
```bash ```bash
sudo gitlab-rake gitlab:artifacts:check BATCH=100 ID_FROM=50 ID_TO=250
sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250 sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250
sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250 sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
``` ```
......
module Gitlab
module Verify
class JobArtifacts < BatchVerifier
def name
'Job artifacts'
end
def describe(object)
"Job artifact: #{object.id}"
end
private
def relation
::Ci::JobArtifact.all
end
def expected_checksum(artifact)
artifact.file_sha256
end
def actual_checksum(artifact)
Digest::SHA256.file(artifact.file.path).hexdigest
end
end
end
end
namespace :gitlab do
namespace :artifacts do
desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts'
task check: :environment do
Gitlab::Verify::RakeTask.run!(Gitlab::Verify::JobArtifacts)
end
end
end
...@@ -35,5 +35,11 @@ FactoryBot.define do ...@@ -35,5 +35,11 @@ FactoryBot.define do
Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain') Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
end end
end end
trait :correct_checksum do
after(:build) do |artifact, evaluator|
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
end
end
end end
end end
require 'spec_helper'
describe Gitlab::Verify::JobArtifacts do
include GitlabVerifyHelpers
it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
let!(:objects) { create_list(:ci_job_artifact, 3, :archive) }
end
describe '#run_batches' do
let(:failures) { collect_failures }
let(:failure) { failures[artifact] }
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
it 'passes artifacts with the correct file' do
expect(failures).to eq({})
end
it 'fails artifacts with a missing file' do
FileUtils.rm_f(artifact.file.path)
expect(failures.keys).to contain_exactly(artifact)
expect(failure).to be_a(Errno::ENOENT)
expect(failure.to_s).to include(artifact.file.path)
end
it 'fails artifacts with a mismatched checksum' do
File.truncate(artifact.file.path, 0)
expect(failures.keys).to contain_exactly(artifact)
expect(failure.to_s).to include('Checksum mismatch')
end
end
end
require 'rake_helper'
describe 'gitlab:artifacts rake tasks' do
describe 'check' do
let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
before do
Rake.application.rake_require('tasks/gitlab/artifacts/check')
stub_env('VERBOSE' => 'true')
end
it 'outputs the integrity check for each batch' do
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Failures: 0/).to_stdout
end
it 'errors out about missing files on the file system' do
FileUtils.rm_f(artifact.file.path)
expect { run_rake_task('gitlab:artifacts:check') }.to output(/No such file.*#{Regexp.quote(artifact.file.path)}/).to_stdout
end
it 'errors out about invalid checksum' do
artifact.update_column(:file_sha256, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum mismatch/).to_stdout
end
it 'errors out about missing checksum' do
artifact.update_column(:file_sha256, nil)
expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum missing/).to_stdout
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment