Commit 6a0bbb5a authored by James Lopez's avatar James Lopez

using shared path for project import uploads and refactored gitlab remove export worker

parent 632113e4
...@@ -47,6 +47,9 @@ v 8.11.0 (unreleased) ...@@ -47,6 +47,9 @@ v 8.11.0 (unreleased)
- Fix RequestProfiler::Middleware error when code is reloaded in development - Fix RequestProfiler::Middleware error when code is reloaded in development
- Catch what warden might throw when profiling requests to re-throw it - Catch what warden might throw when profiling requests to re-throw it
v 8.10.4 (unreleased)
- Fix Import/Export project import not working in HA mode
v 8.10.3 v 8.10.3
- Fix Import/Export issue importing milestones and labels not associated properly. !5426 - Fix Import/Export issue importing milestones and labels not associated properly. !5426
- Fix timing problems running imports on production. !5523 - Fix timing problems running imports on production. !5523
......
...@@ -12,13 +12,14 @@ class Import::GitlabProjectsController < Import::BaseController ...@@ -12,13 +12,14 @@ class Import::GitlabProjectsController < Import::BaseController
return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." }) return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
end end
imported_file = project_params[:file].path + "-import" import_upload_path = Gitlab::ImportExport.import_upload_path(filename: project_params[:file].original_filename)
FileUtils.copy_entry(project_params[:file].path, imported_file) FileUtils.mkdir_p(File.dirname(import_upload_path))
FileUtils.copy_entry(project_params[:file].path, import_upload_path)
@project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id], @project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
current_user, current_user,
File.expand_path(imported_file), import_upload_path,
project_params[:path]).execute project_params[:path]).execute
if @project.saved? if @project.saved?
......
...@@ -378,11 +378,6 @@ class Project < ActiveRecord::Base ...@@ -378,11 +378,6 @@ class Project < ActiveRecord::Base
joins(join_body).reorder('join_note_counts.amount DESC') joins(join_body).reorder('join_note_counts.amount DESC')
end end
# Deletes gitlab project export files older than 24 hours
def remove_gitlab_exports!
Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
end
end end
def repository_storage_path def repository_storage_path
......
class ImportExportCleanUpService
LAST_MODIFIED_TIME_IN_MINUTES = 1440
attr_reader :mmin, :path
def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
@mmin = mmin
@path = Gitlab::ImportExport.storage_path
end
def execute
Gitlab::Metrics.measure(:import_export_clean_up) do
return unless File.directory?(path)
clean_up_export_files
end
end
private
def clean_up_export_files
Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
end
end
class RepositoryArchiveCleanUpService class RepositoryArchiveCleanUpService
LAST_MODIFIED_TIME_IN_MINUTES = 120 LAST_MODIFIED_TIME_IN_MINUTES = 120
attr_reader :mmin, :path
def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES) def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
@mmin = mmin @mmin = mmin
@path = Gitlab.config.gitlab.repository_downloads_path @path = Gitlab.config.gitlab.repository_downloads_path
...@@ -17,8 +19,6 @@ class RepositoryArchiveCleanUpService ...@@ -17,8 +19,6 @@ class RepositoryArchiveCleanUpService
private private
attr_reader :mmin, :path
def clean_up_old_archives def clean_up_old_archives
run(%W(find #{path} -not -path #{path} -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -maxdepth 2 -mmin +#{mmin} -delete)) run(%W(find #{path} -not -path #{path} -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -maxdepth 2 -mmin +#{mmin} -delete))
end end
......
class GitlabRemoveProjectExportWorker class ImportExportProjectCleanupWorker
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options queue: :default sidekiq_options queue: :default
def perform def perform
Project.remove_gitlab_exports! ImportExportCleanUpService.new.execute
end end
end end
...@@ -287,9 +287,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' ...@@ -287,9 +287,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker' Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker' Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
......
...@@ -13,6 +13,10 @@ module Gitlab ...@@ -13,6 +13,10 @@ module Gitlab
File.join(Settings.shared['path'], 'tmp/project_exports') File.join(Settings.shared['path'], 'tmp/project_exports')
end end
def import_upload_path(filename:)
File.join(storage_path, 'uploads', filename)
end
def project_filename def project_filename
"project.json" "project.json"
end end
......
require 'spec_helper'
describe ImportExportCleanUpService, services: true do
describe '#execute' do
let(:service) { described_class.new }
let(:tmp_import_export_folder) { 'tmp/project_exports' }
context 'when the import/export directory does not exist' do
it 'does not remove any archives' do
path = '/invalid/path/'
stub_repository_downloads_path(path)
expect(File).to receive(:directory?).with(path + tmp_import_export_folder).and_return(false).at_least(:once)
expect(service).not_to receive(:clean_up_export_files)
service.execute
end
end
context 'when the import/export directory exists' do
it 'removes old files' do
in_directory_with_files(mtime: 2.days.ago) do |dir, files|
service.execute
files.each { |file| expect(File.exist?(file)).to eq false }
expect(File.directory?(dir)).to eq false
end
end
it 'does not remove new files' do
in_directory_with_files(mtime: 2.hours.ago) do |dir, files|
service.execute
files.each { |file| expect(File.exist?(file)).to eq true }
expect(File.directory?(dir)).to eq true
end
end
end
def in_directory_with_files(mtime:)
Dir.mktmpdir do |tmpdir|
stub_repository_downloads_path(tmpdir)
dir = File.join(tmpdir, tmp_import_export_folder, 'subfolder')
FileUtils.mkdir_p(dir)
files = FileUtils.touch(file_list(dir) + [dir], mtime: mtime)
yield(dir, files)
end
end
def stub_repository_downloads_path(path)
new_shared_settings = Settings.shared.merge('path' => path)
allow(Settings).to receive(:shared).and_return(new_shared_settings)
end
def file_list(dir)
Array.new(5) do |num|
File.join(dir, "random-#{num}.tar.gz")
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment