Commit e83aa920 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'ee-fix-direct-upload-for-old-records' into 'master'

EE: Fix direct_upload when records with null file_store are used

See merge request gitlab-org/gitlab-ee!5414
parents daa555ee 5a11dce7
......@@ -9,16 +9,17 @@ module Ci
belongs_to :project
belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
before_save :update_file_store
mount_uploader :file, JobArtifactUploader
before_save :set_size, if: :file_changed?
after_save :update_project_statistics_after_save, if: :size_changed?
after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed?
after_save :update_file_store
scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
mount_uploader :file, JobArtifactUploader
delegate :exists?, :open, to: :file
enum file_type: {
......@@ -28,7 +29,9 @@ module Ci
}
def update_file_store
self.file_store = file.object_store
# The file.object_store is set during `uploader.store!`
# which happens after object is inserted/updated
self.update_column(:file_store, file.object_store)
end
def self.artifacts_size_for(project)
......
......@@ -13,10 +13,12 @@ class LfsObject < ActiveRecord::Base
mount_uploader :file, LfsObjectUploader
before_save :update_file_store
after_save :update_file_store
def update_file_store
self.file_store = file.object_store
# The file.object_store is set during `uploader.store!`
# which happens after object is inserted/updated
self.update_column(:file_store, file.object_store)
end
def project_allowed_access?(project)
......
......@@ -183,14 +183,6 @@ module ObjectStorage
StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options)
}
end
def default_object_store
if self.object_store_enabled? && self.direct_upload_enabled?
Store::REMOTE
else
Store::LOCAL
end
end
end
# allow to configure and overwrite the filename
......@@ -211,12 +203,13 @@ module ObjectStorage
end
def object_store
@object_store ||= model.try(store_serialization_column) || self.class.default_object_store
# We use Store::LOCAL as null value indicates the local storage
@object_store ||= model.try(store_serialization_column) || Store::LOCAL
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def object_store=(value)
@object_store = value || self.class.default_object_store
@object_store = value || Store::LOCAL
@storage = storage_for(object_store)
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
......@@ -302,6 +295,15 @@ module ObjectStorage
super
end
def store!(new_file = nil)
# when direct upload is enabled, always store on remote storage
if self.class.object_store_enabled? && self.class.direct_upload_enabled?
self.object_store = Store::REMOTE
end
super
end
private
def schedule_background_upload?
......
---
title: Fix direct_upload when records with null file_store are used
merge_request:
author:
type: fixed
......@@ -168,4 +168,43 @@ describe Ci::JobArtifact do
end
end
end
describe 'file is being stored' do
subject { create(:ci_job_artifact, :archive) }
context 'when object has nil store' do
before do
subject.update_column(:file_store, nil)
subject.reload
end
it 'is stored locally' do
expect(subject.file_store).to be(nil)
expect(subject.file).to be_file_storage
expect(subject.file.object_store).to eq(ObjectStorage::Store::LOCAL)
end
end
context 'when existing object has local store' do
it 'is stored locally' do
expect(subject.file_store).to be(ObjectStorage::Store::LOCAL)
expect(subject.file).to be_file_storage
expect(subject.file.object_store).to eq(ObjectStorage::Store::LOCAL)
end
end
context 'when direct upload is enabled' do
before do
stub_artifacts_object_storage(direct_upload: true)
end
context 'when file is stored' do
it 'is stored remotely' do
expect(subject.file_store).to eq(ObjectStorage::Store::REMOTE)
expect(subject.file).not_to be_file_storage
expect(subject.file.object_store).to eq(ObjectStorage::Store::REMOTE)
end
end
end
end
end
......@@ -81,5 +81,44 @@ describe LfsObject do
end
end
end
describe 'file is being stored' do
let(:lfs_object) { create(:lfs_object, :with_file) }
context 'when object has nil store' do
before do
lfs_object.update_column(:file_store, nil)
lfs_object.reload
end
it 'is stored locally' do
expect(lfs_object.file_store).to be(nil)
expect(lfs_object.file).to be_file_storage
expect(lfs_object.file.object_store).to eq(ObjectStorage::Store::LOCAL)
end
end
context 'when existing object has local store' do
it 'is stored locally' do
expect(lfs_object.file_store).to be(ObjectStorage::Store::LOCAL)
expect(lfs_object.file).to be_file_storage
expect(lfs_object.file.object_store).to eq(ObjectStorage::Store::LOCAL)
end
end
context 'when direct upload is enabled' do
before do
stub_lfs_object_storage(direct_upload: true)
end
context 'when file is stored' do
it 'is stored remotely' do
expect(lfs_object.file_store).to eq(ObjectStorage::Store::REMOTE)
expect(lfs_object.file).not_to be_file_storage
expect(lfs_object.file.object_store).to eq(ObjectStorage::Store::REMOTE)
end
end
end
end
end
end
......@@ -75,36 +75,8 @@ describe ObjectStorage do
expect(object).to receive(:file_store).and_return(nil)
end
context 'when object storage is enabled' do
context 'when direct uploads are enabled' do
before do
stub_uploads_object_storage(uploader_class, enabled: true, direct_upload: true)
end
it "uses Store::REMOTE" do
is_expected.to eq(described_class::Store::REMOTE)
end
end
context 'when direct uploads are disabled' do
before do
stub_uploads_object_storage(uploader_class, enabled: true, direct_upload: false)
end
it "uses Store::LOCAL" do
is_expected.to eq(described_class::Store::LOCAL)
end
end
end
context 'when object storage is disabled' do
before do
stub_uploads_object_storage(uploader_class, enabled: false)
end
it "uses Store::LOCAL" do
is_expected.to eq(described_class::Store::LOCAL)
end
it "uses Store::LOCAL" do
is_expected.to eq(described_class::Store::LOCAL)
end
end
......@@ -549,6 +521,72 @@ describe ObjectStorage do
end
end
context 'when local file is used' do
let(:temp_file) { Tempfile.new("test") }
before do
FileUtils.touch(temp_file)
end
after do
FileUtils.rm_f(temp_file)
end
context 'when valid file is used' do
context 'when valid file is specified' do
let(:uploaded_file) { temp_file }
context 'when object storage and direct upload is specified' do
before do
stub_uploads_object_storage(uploader_class, enabled: true, direct_upload: true)
end
context 'when file is stored' do
subject do
uploader.store!(uploaded_file)
end
it 'file to be remotely stored in permament location' do
subject
expect(uploader).to be_exists
expect(uploader).not_to be_cached
expect(uploader).not_to be_file_storage
expect(uploader.path).not_to be_nil
expect(uploader.path).not_to include('tmp/upload')
expect(uploader.path).not_to include('tmp/cache')
expect(uploader.object_store).to eq(described_class::Store::REMOTE)
end
end
end
context 'when object storage and direct upload is not used' do
before do
stub_uploads_object_storage(uploader_class, enabled: true, direct_upload: false)
end
context 'when file is stored' do
subject do
uploader.store!(uploaded_file)
end
it 'file to be remotely stored in permament location' do
subject
expect(uploader).to be_exists
expect(uploader).not_to be_cached
expect(uploader).to be_file_storage
expect(uploader.path).not_to be_nil
expect(uploader.path).not_to include('tmp/upload')
expect(uploader.path).not_to include('tmp/cache')
expect(uploader.object_store).to eq(described_class::Store::LOCAL)
end
end
end
end
end
end
context 'when remote file is used' do
let(:temp_file) { Tempfile.new("test") }
......@@ -602,9 +640,9 @@ describe ObjectStorage do
expect(uploader).to be_exists
expect(uploader).to be_cached
expect(uploader).not_to be_file_storage
expect(uploader.path).not_to be_nil
expect(uploader.path).not_to include('tmp/cache')
expect(uploader.url).not_to be_nil
expect(uploader.path).not_to include('tmp/cache')
expect(uploader.object_store).to eq(described_class::Store::REMOTE)
end
......@@ -619,6 +657,7 @@ describe ObjectStorage do
expect(uploader).to be_exists
expect(uploader).not_to be_cached
expect(uploader).not_to be_file_storage
expect(uploader.path).not_to be_nil
expect(uploader.path).not_to include('tmp/upload')
expect(uploader.path).not_to include('tmp/cache')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment