Commit c4c8ca04 authored by Kamil Trzcinski's avatar Kamil Trzcinski Committed by James Edwards-Jones

Added support for zip archives in pages

The ZIP archive size is calculated from artifacts metadata that should get uploaded for new artifacts
parent 6e70870a
...@@ -2,6 +2,7 @@ module Projects ...@@ -2,6 +2,7 @@ module Projects
class UpdatePagesService < BaseService class UpdatePagesService < BaseService
BLOCK_SIZE = 32.kilobytes BLOCK_SIZE = 32.kilobytes
MAX_SIZE = 1.terabyte MAX_SIZE = 1.terabyte
SITE_PATH = 'public/'
attr_reader :build attr_reader :build
...@@ -60,13 +61,42 @@ module Projects ...@@ -60,13 +61,42 @@ module Projects
end end
def extract_archive!(temp_path) def extract_archive!(temp_path)
if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz')
extract_tar_archive!(temp_path)
elsif artifacts.ends_with?('.zip')
extract_zip_archive!(temp_path)
else
raise 'unsupported artifacts format'
end
end
def extract_tar_archive!(temp_path)
results = Open3.pipeline(%W(gunzip -c #{artifacts}), results = Open3.pipeline(%W(gunzip -c #{artifacts}),
%W(dd bs=#{BLOCK_SIZE} count=#{blocks}), %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
%W(tar -x -C #{temp_path} public/), %W(tar -x -C #{temp_path} #{SITE_PATH}),
err: '/dev/null') err: '/dev/null')
raise 'pages failed to extract' unless results.compact.all?(&:success?) raise 'pages failed to extract' unless results.compact.all?(&:success?)
end end
def extract_zip_archive!(temp_path)
raise 'missing artifacts metadata' unless build.artifacts_metadata?
# Calculate page size after extract
public_entry = build.artifacts_metadata_entry(SITE_PATH, recursive: true)
if public_entry.total_size > max_size
raise "artifacts for pages are too large: #{total_size}"
end
# Requires UnZip at least 6.00 Info-ZIP.
# -n never overwrite existing files
# We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
site_path = File.join(SITE_PATH, '*')
unless system(*%W(unzip -n #{artifacts} #{site_path} -d #{temp_path}))
raise 'pages failed to extract'
end
end
def deploy_page!(archive_public_path) def deploy_page!(archive_public_path)
# Do atomic move of pages # Do atomic move of pages
# Move and removal may not be atomic, but they are significantly faster then extracting and removal # Move and removal may not be atomic, but they are significantly faster then extracting and removal
...@@ -91,10 +121,11 @@ module Projects ...@@ -91,10 +121,11 @@ module Projects
def blocks def blocks
# Calculate dd parameters: we limit the size of pages # Calculate dd parameters: we limit the size of pages
max_size = current_application_settings.max_pages_size.megabytes 1 + max_size / BLOCK_SIZE
max_size ||= MAX_SIZE end
blocks = 1 + max_size / BLOCK_SIZE
blocks def max_size
current_application_settings.max_pages_size.megabytes || MAX_SIZE
end end
def tmp_path def tmp_path
......
...@@ -4,9 +4,7 @@ describe Projects::UpdatePagesService do ...@@ -4,9 +4,7 @@ describe Projects::UpdatePagesService do
let(:project) { create :project } let(:project) { create :project }
let(:commit) { create :ci_commit, project: project, sha: project.commit('HEAD').sha } let(:commit) { create :ci_commit, project: project, sha: project.commit('HEAD').sha }
let(:build) { create :ci_build, commit: commit, ref: 'HEAD' } let(:build) { create :ci_build, commit: commit, ref: 'HEAD' }
let(:file) { fixture_file_upload(Rails.root + 'spec/fixtures/pages.tar.gz', 'application/octet-stream') } let(:invalid_file) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png') }
let(:empty_file) { fixture_file_upload(Rails.root + 'spec/fixtures/pages_empty.tar.gz', 'application/octet-stream') }
let(:invalid_file) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'application/octet-stream') }
subject { described_class.new(project, build) } subject { described_class.new(project, build) }
...@@ -14,8 +12,19 @@ describe Projects::UpdatePagesService do ...@@ -14,8 +12,19 @@ describe Projects::UpdatePagesService do
project.remove_pages project.remove_pages
end end
context 'for valid file' do %w(tar.gz zip).each do |format|
before { build.update_attributes(artifacts_file: file) } context "for valid #{format}" do
let(:file) { fixture_file_upload(Rails.root + "spec/fixtures/pages.#{format}") }
let(:empty_file) { fixture_file_upload(Rails.root + "spec/fixtures/pages_empty.#{format}") }
let(:metadata) do
filename = Rails.root + "spec/fixtures/pages.#{format}.meta"
fixture_file_upload(filename) if File.exists?(filename)
end
before do
build.update_attributes(artifacts_file: file)
build.update_attributes(artifacts_metadata: metadata)
end
it 'succeeds' do it 'succeeds' do
expect(project.pages_url).to be_nil expect(project.pages_url).to be_nil
...@@ -36,6 +45,18 @@ describe Projects::UpdatePagesService do ...@@ -36,6 +45,18 @@ describe Projects::UpdatePagesService do
project.destroy project.destroy
expect(Dir.exist?(project.public_pages_path)).to be_falsey expect(Dir.exist?(project.public_pages_path)).to be_falsey
end end
it 'fails if sha on branch is not latest' do
commit.update_attributes(sha: 'old_sha')
build.update_attributes(artifacts_file: file)
expect(execute).to_not eq(:success)
end
it 'fails for empty file fails' do
build.update_attributes(artifacts_file: empty_file)
expect(execute).to_not eq(:success)
end
end
end end
it 'fails to remove project pages when no pages is deployed' do it 'fails to remove project pages when no pages is deployed' do
...@@ -48,22 +69,11 @@ describe Projects::UpdatePagesService do ...@@ -48,22 +69,11 @@ describe Projects::UpdatePagesService do
expect(execute).to_not eq(:success) expect(execute).to_not eq(:success)
end end
it 'fails for empty file fails' do
build.update_attributes(artifacts_file: empty_file)
expect(execute).to_not eq(:success)
end
it 'fails for invalid archive' do it 'fails for invalid archive' do
build.update_attributes(artifacts_file: invalid_file) build.update_attributes(artifacts_file: invalid_file)
expect(execute).to_not eq(:success) expect(execute).to_not eq(:success)
end end
it 'fails if sha on branch is not latest' do
commit.update_attributes(sha: 'old_sha')
build.update_attributes(artifacts_file: file)
expect(execute).to_not eq(:success)
end
def execute def execute
subject.execute[:status] subject.execute[:status]
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment