Commit 82007530 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'da-verify-integrity-of-uploaded-files' into 'master'

Resolve "Verify integrity of uploaded files"

Closes #39949

See merge request gitlab-org/gitlab-ce!16297
parents af725035 d270693c
---
title: Add rake task to check integrity of uploaded files
merge_request:
author:
type: added
...@@ -76,6 +76,39 @@ Example output: ...@@ -76,6 +76,39 @@ Example output:
![gitlab:user:check_repos output](../img/raketasks/check_repos_output.png) ![gitlab:user:check_repos output](../img/raketasks/check_repos_output.png)
## Uploaded Files Integrity
The uploads check Rake task will loop through all uploads in the database
and run two checks to determine the integrity of each file:
1. Check if the file exist on the file system.
1. Check if the checksum of the file on the file system matches the checksum in the database.
**Omnibus Installation**
```
sudo gitlab-rake gitlab:uploads:check
```
**Source Installation**
```bash
sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
```
This task also accepts some environment variables which you can use to override
certain values:
Variable | Type | Description
-------- | ---- | -----------
`BATCH` | integer | Specifies the size of the batch. Defaults to 200.
`ID_FROM` | integer | Specifies the ID to start from, inclusive of the value.
`ID_TO` | integer | Specifies the ID value to end at, inclusive of the value.
```bash
sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
```
## LDAP Check ## LDAP Check
The LDAP check Rake task will test the bind_dn and password credentials The LDAP check Rake task will test the bind_dn and password credentials
......
namespace :gitlab do
namespace :uploads do
desc 'GitLab | Uploads | Check integrity of uploaded files'
task check: :environment do
puts 'Checking integrity of uploaded files'
uploads_batches do |batch|
batch.each do |upload|
puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green)
if upload.exist?
check_checksum(upload)
else
puts " * File does not exist on the file system".color(:red)
end
end
end
puts 'Done!'
end
def batch_size
ENV.fetch('BATCH', 200).to_i
end
def calculate_checksum(absolute_path)
Digest::SHA256.file(absolute_path).hexdigest
end
def check_checksum(upload)
checksum = calculate_checksum(upload.absolute_path)
if checksum != upload.checksum
puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red)
end
end
def uploads_batches(&block)
Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
yield relation
end
end
end
end
require 'rake_helper'
describe 'gitlab:uploads rake tasks' do
describe 'check' do
let!(:upload) { create(:upload, path: Rails.root.join('spec/fixtures/banana_sample.gif')) }
before do
Rake.application.rake_require 'tasks/gitlab/uploads'
end
it 'outputs the integrity check for each uploaded file' do
expect { run_rake_task('gitlab:uploads:check') }.to output(/Checking file \(#{upload.id}\): #{Regexp.quote(upload.absolute_path)}/).to_stdout
end
it 'errors out about missing files on the file system' do
create(:upload)
expect { run_rake_task('gitlab:uploads:check') }.to output(/File does not exist on the file system/).to_stdout
end
it 'errors out about invalid checksum' do
upload.update_column(:checksum, '01a3156db2cf4f67ec823680b40b7302f89ab39179124ad219f94919b8a1769e')
expect { run_rake_task('gitlab:uploads:check') }.to output(/File checksum \(9e697aa09fe196909813ee36103e34f721fe47a5fdc8aac0e4e4ac47b9b38282\) does not match the one in the database \(#{upload.checksum}\)/).to_stdout
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment