Commit 2cfd0b59 authored by Douwe Maan's avatar Douwe Maan

Archive repositories in background worker.

parent 33a8f53f
web: bundle exec unicorn_rails -p ${PORT:="3000"} -E ${RAILS_ENV:="development"} -c ${UNICORN_CONFIG:="config/unicorn.rb"} web: bundle exec unicorn_rails -p ${PORT:="3000"} -E ${RAILS_ENV:="development"} -c ${UNICORN_CONFIG:="config/unicorn.rb"}
worker: bundle exec sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default worker: bundle exec sidekiq -q post_receive -q mailer -q archive_repo -q system_hook -q project_web_hook -q gitlab_shell -q common -q default
...@@ -15,14 +15,18 @@ class Projects::RepositoriesController < Projects::ApplicationController ...@@ -15,14 +15,18 @@ class Projects::RepositoriesController < Projects::ApplicationController
render_404 and return render_404 and return
end end
file_path = ArchiveRepositoryService.new.execute(@project, params[:ref], params[:format]) begin
file_path = ArchiveRepositoryService.new(@project, params[:ref], params[:format]).execute
rescue
return render_404
end
if file_path if file_path
# Send file to user # Send file to user
response.headers["Content-Length"] = File.open(file_path).size.to_s response.headers["Content-Length"] = File.open(file_path).size.to_s
send_file file_path send_file file_path
else else
render_404 redirect_to request.fullpath
end end
end end
end end
...@@ -267,6 +267,9 @@ class Repository ...@@ -267,6 +267,9 @@ class Repository
# Remove archives older than 2 hours # Remove archives older than 2 hours
def clean_old_archives def clean_old_archives
repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
return unless File.directory?(repository_downloads_path)
Gitlab::Popen.popen(%W(find #{repository_downloads_path} -not -path #{repository_downloads_path} -mmin +120 -delete)) Gitlab::Popen.popen(%W(find #{repository_downloads_path} -not -path #{repository_downloads_path} -mmin +120 -delete))
end end
......
class ArchiveRepositoryService class ArchiveRepositoryService
def execute(project, ref, format) attr_reader :project, :ref, :format
storage_path = Gitlab.config.gitlab.repository_downloads_path
unless File.directory?(storage_path) def initialize(project, ref, format)
FileUtils.mkdir_p(storage_path) format ||= 'tar.gz'
@project, @ref, @format = project, ref, format
end
def execute
project.repository.clean_old_archives
raise "No archive file path" unless file_path
return file_path if archived?
unless archiving?
RepositoryArchiveWorker.perform_async(project.id, ref, format)
end end
format ||= 'tar.gz' archived = wait_until_archived
repository = project.repository
repository.clean_old_archives file_path if archived
repository.archive_repo(ref, storage_path, format.downcase) end
private
def storage_path
Gitlab.config.gitlab.repository_downloads_path
end
def archive_args
@archive_args ||= [ref, storage_path, format.downcase]
end
def file_path
@file_path ||= project.repository.archive_file_path(*archive_args)
end
def pid_file_path
@pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
end
def archived?
File.exist?(file_path)
end
def archiving?
File.exist?(pid_file_path)
end
def wait_until_archived
timeout = 5.0
t1 = Time.now
begin
sleep 0.1
success = archived?
t2 = Time.now
end until success || t2 - t1 >= timeout
success
end end
end end
class RepositoryArchiveWorker
include Sidekiq::Worker
sidekiq_options queue: :archive_repo
attr_accessor :project, :ref, :format
def perform(project_id, ref, format)
@project = Project.find(project_id)
@ref, @format = ref, format
repository = project.repository
repository.clean_old_archives
return if archived? || archiving?
repository.archive_repo(*archive_args)
end
private
def storage_path
Gitlab.config.gitlab.repository_downloads_path
end
def archive_args
@archive_args ||= [ref, storage_path, format.downcase]
end
def file_path
@file_path ||= project.repository.archive_file_path(*archive_args)
end
def pid_file_path
@pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
end
def archived?
File.exist?(file_path)
end
def archiving?
File.exist?(pid_file_path)
end
end
...@@ -133,10 +133,11 @@ module API ...@@ -133,10 +133,11 @@ module API
authorize! :download_code, user_project authorize! :download_code, user_project
begin begin
file_path = ArchiveRepositoryService.new.execute( file_path = ArchiveRepositoryService.new(
user_project, user_project,
params[:sha], params[:sha],
params[:format]) params[:format]
).execute
rescue rescue
not_found!('File') not_found!('File')
end end
...@@ -149,7 +150,7 @@ module API ...@@ -149,7 +150,7 @@ module API
env['api.format'] = :binary env['api.format'] = :binary
present data present data
else else
not_found!('File') redirect request.fullpath
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment