Merge branch 'handle-object-storage-errors' into 'master'

Return 503 to Runner when object storage is unavailable

See merge request gitlab-org/gitlab!25822

app/services/ci/create_job_artifacts_service.rb

 # frozen_string_literal: true
 
 module Ci
-  class CreateJobArtifactsService
+  class CreateJobArtifactsService < ::BaseService
     ArtifactsExistError = Class.new(StandardError)
+    OBJECT_STORAGE_ERRORS = [
+      Errno::EIO,
+      Google::Apis::ServerError,
+      Signet::RemoteServerError
+    ].freeze
 
     def execute(job, artifacts_file, params, metadata_file: nil)
       expire_in = params['expire_in'] ||
@@ -26,18 +31,20 @@ module Ci
           expire_in: expire_in)
       end
 
-      job.update(artifacts_expire_in: expire_in)
-    rescue ActiveRecord::RecordNotUnique => error
-      return true if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
+      if job.update(artifacts_expire_in: expire_in)
+        success
+      else
+        error(job.errors.messages, :bad_request)
+      end
 
-      Gitlab::ErrorTracking.track_exception(error,
-        job_id: job.id,
-        project_id: job.project_id,
-        uploading_type: params['artifact_type']
-      )
+    rescue ActiveRecord::RecordNotUnique => error
+      return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
 
-      job.errors.add(:base, 'another artifact of the same type already exists')
-      false
+      track_exception(error, job, params)
+      error('another artifact of the same type already exists', :bad_request)
+    rescue *OBJECT_STORAGE_ERRORS => error
+      track_exception(error, job, params)
+      error(error.message, :service_unavailable)
     end
 
     private
@@ -48,5 +55,13 @@ module Ci
 
       existing_artifact.file_sha256 == artifacts_file.sha256
     end
+
+    def track_exception(error, job, params)
+      Gitlab::ErrorTracking.track_exception(error,
+        job_id: job.id,
+        project_id: job.project_id,
+        uploading_type: params['artifact_type']
+      )
+    end
   end
 end

changelogs/unreleased/handle-object-storage-errors.yml

+---
+title: Return 503 to the Runner when the object storage is unavailable
+merge_request: 25822
+author:
+type: fixed

lib/api/runner.rb

@@ -283,10 +283,12 @@ module API
         bad_request!('Missing artifacts file!') unless artifacts
         file_too_large! unless artifacts.size < max_artifacts_size(job)
 
-        if Ci::CreateJobArtifactsService.new.execute(job, artifacts, params, metadata_file: metadata)
+        result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata)
+
+        if result[:status] == :success
           status :created
         else
-          render_validation_error!(job)
+          render_api_error!(result[:message], result[:http_status])
         end
       end
 

spec/requests/api/runner_spec.rb

@@ -1979,6 +1979,21 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
           end
         end
 
+        context 'when object storage throws errors' do
+          let(:params) { { artifact_type: :archive, artifact_format: :zip } }
+
+          it 'does not store artifacts' do
+            allow_next_instance_of(JobArtifactUploader) do |uploader|
+              allow(uploader).to receive(:store!).and_raise(Errno::EIO)
+            end
+
+            upload_artifacts(file_upload, headers_with_token, params)
+
+            expect(response).to have_gitlab_http_status(:service_unavailable)
+            expect(job.reload.job_artifacts_archive).to be_nil
+          end
+        end
+
         context 'when artifacts are being stored outside of tmp path' do
           let(:new_tmpdir) { Dir.mktmpdir }
 

spec/services/ci/create_job_artifacts_service_spec.rb

@@ -3,8 +3,9 @@
 require 'spec_helper'
 
 describe Ci::CreateJobArtifactsService do
-  let(:service) { described_class.new }
-  let(:job) { create(:ci_build) }
+  let_it_be(:project) { create(:project) }
+  let(:service) { described_class.new(project) }
+  let(:job) { create(:ci_build, project: project) }
   let(:artifacts_sha256) { '0' * 64 }
   let(:metadata_file) { nil }
 
@@ -64,7 +65,7 @@ describe Ci::CreateJobArtifactsService do
         it 'sets expiration date according to application settings' do
           expected_expire_at = 1.day.from_now
 
-          expect(subject).to be_truthy
+          expect(subject).to match(a_hash_including(status: :success))
           archive_artifact, metadata_artifact = job.job_artifacts.last(2)
 
           expect(job.artifacts_expire_at).to be_within(1.minute).of(expected_expire_at)
@@ -80,7 +81,7 @@ describe Ci::CreateJobArtifactsService do
           it 'sets expiration date according to the parameter' do
             expected_expire_at = 2.hours.from_now
 
-            expect(subject).to be_truthy
+            expect(subject).to match(a_hash_including(status: :success))
             archive_artifact, metadata_artifact = job.job_artifacts.last(2)
 
             expect(job.artifacts_expire_at).to be_within(1.minute).of(expected_expire_at)
@@ -101,21 +102,50 @@ describe Ci::CreateJobArtifactsService do
 
         it 'ignores the changes' do
           expect { subject }.not_to change { Ci::JobArtifact.count }
-          expect(subject).to be_truthy
+          expect(subject).to match(a_hash_including(status: :success))
         end
       end
 
       context 'when sha256 of uploading artifact is different than the existing one' do
         let(:existing_sha256) { '1' * 64 }
 
-        it 'returns false and logs the error' do
+        it 'returns error status' do
           expect(Gitlab::ErrorTracking).to receive(:track_exception).and_call_original
 
           expect { subject }.not_to change { Ci::JobArtifact.count }
-          expect(subject).to be_falsey
-          expect(job.errors[:base]).to contain_exactly('another artifact of the same type already exists')
+          expect(subject).to match(
+            a_hash_including(http_status: :bad_request,
+              message: 'another artifact of the same type already exists',
+              status: :error))
         end
       end
     end
+
+    shared_examples 'rescues object storage error' do |klass, message, expected_message|
+      it "handles #{klass}" do
+        allow_next_instance_of(JobArtifactUploader) do |uploader|
+          allow(uploader).to receive(:store!).and_raise(klass, message)
+        end
+
+        expect(Gitlab::ErrorTracking)
+          .to receive(:track_exception)
+          .and_call_original
+
+        expect(subject).to match(
+          a_hash_including(
+            http_status: :service_unavailable,
+            message: expected_message || message,
+            status: :error))
+      end
+    end
+
+    it_behaves_like 'rescues object storage error',
+      Errno::EIO, 'some/path', 'Input/output error - some/path'
+
+    it_behaves_like 'rescues object storage error',
+      Google::Apis::ServerError, 'Server error'
+
+    it_behaves_like 'rescues object storage error',
+      Signet::RemoteServerError, 'The service is currently unavailable'
   end
 end