Add spec that proves trace can be recovered even if it had redis outage

lib/api/helpers/runner.rb

@@ -52,6 +52,7 @@ module API
       end
 
       def job_token_valid?(job)
+        # binding.pry
         token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
         token && job.valid_token?(token)
       end

spec/requests/api/runner_spec.rb

@@ -2,6 +2,7 @@ require 'spec_helper'
 
 describe API::Runner, :clean_gitlab_redis_shared_state do
   include StubGitlabCalls
+  include ChunkedIOHelpers
 
   let(:registration_token) { 'abcdefg123456' }
 
@@ -865,6 +866,29 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
             expect(response.status).to eq(403)
           end
         end
+
+        context 'when redis had an outage' do
+          it "recovers" do
+            # GitLab-Runner patchs
+            patch_the_trace
+            expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
+
+            # GitLab-Rails enxounters an outage on Redis
+            redis_shared_state_outage!
+            expect(job.reload.trace.raw).to eq ''
+
+            # GitLab-Runner patchs
+            patch_the_trace('hello', headers.merge({ 'Content-Range' => "28-32" }))
+            expect(response.status).to eq 202
+            expect(response.header).to have_key 'Range'
+            expect(response.header['Range']).to eq '0-0'
+            expect(job.reload.trace.raw).to eq ''
+
+            # GitLab-Runner re-patchs
+            patch_the_trace('BUILD TRACE appended appended hello')
+            expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended hello'
+          end
+        end
       end
 
       context 'when Runner makes a force-patch' do

spec/support/chunked_io/chunked_io_helpers.rb

@@ -8,4 +8,9 @@ module ChunkedIOHelpers
     stub_const('Ci::JobTraceChunk::CHUNK_SIZE', size)
     stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size)
   end
+
+  def redis_shared_state_outage!
+    Gitlab::Redis::SharedState.with(&:flushall)
+    Sidekiq.redis(&:flushall)
+  end
 end