Commit b96fa309 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'lfs_default_and_backup' into 'master'

Lfs default and backup

Part of gitlab-org/gitlab-ce#2955

See merge request !1823
parents a154c934 2219743d
...@@ -126,7 +126,7 @@ production: &base ...@@ -126,7 +126,7 @@ production: &base
## Git LFS ## Git LFS
lfs: lfs:
enabled: false enabled: true
# The location where LFS objects are stored (default: shared/lfs-objects). # The location where LFS objects are stored (default: shared/lfs-objects).
# storage_path: shared/lfs-objects # storage_path: shared/lfs-objects
......
...@@ -203,7 +203,7 @@ Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mail ...@@ -203,7 +203,7 @@ Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mail
# Git LFS # Git LFS
# #
Settings['lfs'] ||= Settingslogic.new({}) Settings['lfs'] ||= Settingslogic.new({})
Settings.lfs['enabled'] = false if Settings.lfs['enabled'].nil? Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil?
Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root) Settings.lfs['storage_path'] = File.expand_path(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"), Rails.root)
# #
......
...@@ -29,7 +29,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ...@@ -29,7 +29,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
``` ```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts). uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts), lfs (LFS objects).
Use a comma to specify several options at the same time. Use a comma to specify several options at the same time.
``` ```
......
require 'backup/files'
module Backup
class Lfs < Files
def initialize
super('lfs', Settings.lfs.storage_path)
end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end
end
...@@ -154,7 +154,7 @@ module Backup ...@@ -154,7 +154,7 @@ module Backup
end end
def archives_to_backup def archives_to_backup
%w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
end end
def folders_to_backup def folders_to_backup
......
...@@ -13,6 +13,7 @@ namespace :gitlab do ...@@ -13,6 +13,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke
Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke
Rake::Task["gitlab:backup:lfs:create"].invoke
backup = Backup::Manager.new backup = Backup::Manager.new
backup.pack backup.pack
...@@ -34,6 +35,7 @@ namespace :gitlab do ...@@ -34,6 +35,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs")
Rake::Task["gitlab:shell:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup backup.cleanup
...@@ -134,6 +136,25 @@ namespace :gitlab do ...@@ -134,6 +136,25 @@ namespace :gitlab do
end end
end end
namespace :lfs do
task create: :environment do
$progress.puts "Dumping lfs objects ... ".blue
if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
$progress.puts "[SKIPPED]".cyan
else
Backup::Lfs.new.dump
$progress.puts "done".green
end
end
task restore: :environment do
$progress.puts "Restoring lfs objects ... ".blue
Backup::Lfs.new.restore
$progress.puts "done".green
end
end
def configure_cron_mode def configure_cron_mode
if ENV['CRON'] if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a # We need an object we can say 'puts' and 'print' to; let's use a
......
...@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
end end
def reenable_backup_sub_tasks def reenable_backup_sub_tasks
%w{db repo uploads builds artifacts}.each do |subtask| %w{db repo uploads builds artifacts lfs}.each do |subtask|
Rake::Task["gitlab:backup:#{subtask}:create"].reenable Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end end
end end
...@@ -49,7 +49,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -49,7 +49,7 @@ describe 'gitlab:app namespace rake task' do
to raise_error(SystemExit) to raise_error(SystemExit)
end end
it 'should invoke restoration on mach' do it 'should invoke restoration on match' do
allow(YAML).to receive(:load_file). allow(YAML).to receive(:load_file).
and_return({ gitlab_version: gitlab_version }) and_return({ gitlab_version: gitlab_version })
expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke)
...@@ -57,6 +57,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -57,6 +57,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end end
...@@ -114,7 +115,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -114,7 +115,7 @@ describe 'gitlab:app namespace rake task' do
it 'should set correct permissions on the tar contents' do it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen( tar_contents, exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz} %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz}
) )
expect(exit_status).to eq(0) expect(exit_status).to eq(0)
expect(tar_contents).to match('db/') expect(tar_contents).to match('db/')
...@@ -122,12 +123,13 @@ describe 'gitlab:app namespace rake task' do ...@@ -122,12 +123,13 @@ describe 'gitlab:app namespace rake task' do
expect(tar_contents).to match('repositories/') expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz') expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/) expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
end end
it 'should delete temp directories' do it 'should delete temp directories' do
temp_dirs = Dir.glob( temp_dirs = Dir.glob(
File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}') File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs}')
) )
expect(temp_dirs).to be_empty expect(temp_dirs).to be_empty
...@@ -163,13 +165,14 @@ describe 'gitlab:app namespace rake task' do ...@@ -163,13 +165,14 @@ describe 'gitlab:app namespace rake task' do
it "does not contain skipped item" do it "does not contain skipped item" do
tar_contents, _exit_status = Gitlab::Popen.popen( tar_contents, _exit_status = Gitlab::Popen.popen(
%W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz} %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz}
) )
expect(tar_contents).to match('db/') expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz') expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
expect(tar_contents).not_to match('repositories/') expect(tar_contents).not_to match('repositories/')
end end
...@@ -183,6 +186,7 @@ describe 'gitlab:app namespace rake task' do ...@@ -183,6 +186,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:uploads:restore"]).not_to receive :invoke expect(Rake::Task["gitlab:backup:uploads:restore"]).not_to receive :invoke
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:lfs:restore"]).to receive :invoke
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment