Merge branch 'feature/sm/artifacts-trace-ee' into 'master'

EE: Trace as artifacts (FileStorage and ObjectStorage)

Closes #4171

See merge request gitlab-org/gitlab-ee!4258

app/controllers/projects/jobs_controller.rb

 class Projects::JobsController < Projects::ApplicationController
+  prepend EE::Projects::JobsController
+
   before_action :build, except: [:index, :cancel_all]
 
   before_action :authorize_read_build!,

app/models/ci/build.rb

@@ -20,6 +20,7 @@ module Ci
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment

app/models/ci/job_artifact.rb

@@ -16,9 +16,12 @@ module Ci
       end
     end
 
+    delegate :open, :exists?, to: :file
+
     enum file_type: {
       archive: 1,
-      metadata: 2
+      metadata: 2,
+      trace: 3
     }
 
     def self.artifacts_size_for(project)

app/models/concerns/artifact_migratable.rb

@@ -39,7 +39,6 @@ module ArtifactMigratable
   end
 
   def artifacts_size
-    read_attribute(:artifacts_size).to_i +
-      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+    read_attribute(:artifacts_size).to_i + job_artifacts.sum(:size).to_i
   end
 end

app/services/ci/create_trace_artifact_service.rb

+module Ci
+  class CreateTraceArtifactService < BaseService
+    def execute(job)
+      return if job.job_artifacts_trace
+
+      job.trace.read do |stream|
+        if stream.file?
+          job.create_job_artifacts_trace!(
+            project: job.project,
+            file_type: :trace,
+            file: stream)
+        end
+      end
+    end
+  end
+end

app/uploaders/job_artifact_uploader.rb

 class JobArtifactUploader < GitlabUploader
+  prepend EE::JobArtifactUploader
   extend Workhorse::UploadPath
   include ObjectStorage::Concern
 
@@ -14,6 +15,12 @@ class JobArtifactUploader < GitlabUploader
     dynamic_segment
   end
 
+  def open
+    raise 'Only File System is supported' unless file_storage?
+
+    File.open(path, "rb") if path
+  end
+
   private
 
   def dynamic_segment

app/workers/all_queues.yml

@@ -43,6 +43,7 @@
 - pipeline_creation:run_pipeline_schedule
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
+- pipeline_default:create_trace_artifact
 - pipeline_default:pipeline_metrics
 - pipeline_default:pipeline_notification
 - pipeline_default:update_head_pipeline_for_merge_request

app/workers/build_finished_worker.rb

@@ -6,9 +6,13 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
-      BuildTraceSectionsWorker.perform_async(build.id)
+      # We execute that in sync as this access the files in order to access local file, and reduce IO
+      BuildTraceSectionsWorker.new.perform(build.id)
       BuildCoverageWorker.new.perform(build.id)
-      BuildHooksWorker.new.perform(build.id)
+
+      # We execute that async as this are two indepentent operations that can be executed after TraceSections and Coverage
+      BuildHooksWorker.perform_async(build.id)
+      CreateTraceArtifactWorker.perform_async(build.id)
     end
   end
 end

app/workers/create_trace_artifact_worker.rb

+class CreateTraceArtifactWorker
+  include ApplicationWorker
+  include PipelineQueue
+
+  def perform(job_id)
+    Ci::Build.preload(:project, :user).find_by(id: job_id).try do |job|
+      Ci::CreateTraceArtifactService.new(job.project, job.user).execute(job)
+    end
+  end
+end

changelogs/unreleased/feature-sm-artifacts-trace.yml

+---
+title: Save traces as artifacts
+merge_request: 16702
+author:
+type: changed

doc/development/file_storage.md

@@ -16,7 +16,7 @@ There are many places where file uploading is used, according to contexts:
   - Project avatars
   - Issues/MR/Notes Markdown attachments
   - Issues/MR/Notes Legacy Markdown attachments
-  - CI Build Artifacts
+  - CI Artifacts (archive, metadata, trace)
   - LFS Objects
 
 
@@ -35,7 +35,7 @@ they are still not 100% standardized. You can see them below:
 | Project avatars                       | yes    | uploads/-/system/project/avatar/:id/:filename               | `AvatarUploader`       | Project    |
 | Issues/MR/Notes Markdown attachments        | yes    | uploads/:project_path_with_namespace/:random_hex/:filename  | `FileUploader`         | Project    |
 | Issues/MR/Notes Legacy Markdown attachments | no     | uploads/-/system/note/attachment/:id/:filename              | `AttachmentUploader`   | Note       |
-| CI Artifacts (CE)                     | yes    | shared/artifacts/:year_:month/:project_id/:id               | `ArtifactUploader`     | Ci::Build  |
+| CI Artifacts (CE)                     | yes    | shared/artifacts/:disk_hash[0..1]/:disk_hash[2..3]/:disk_hash/:year_:month_:date/:job_id/:job_artifact_id (:disk_hash is SHA256 digest of project_id) | `JobArtifactUploader`  | Ci::JobArtifact  |
 | LFS Objects  (CE)                     | yes    | shared/lfs-objects/:hex/:hex/:object_hash                   | `LfsObjectUploader`    | LfsObject  |
 
 CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader`

ee/app/controllers/ee/projects/jobs_controller.rb

+module EE
+  module Projects
+    module JobsController
+      extend ActiveSupport::Concern
+      include SendFileUpload
+
+      def raw
+        if trace_artifact_file
+          send_upload(trace_artifact_file,
+                      send_params: raw_send_params,
+                      redirect_params: raw_redirect_params)
+        else
+          super
+        end
+      end
+
+      private
+
+      def raw_send_params
+        { type: 'text/plain; charset=utf-8', disposition: 'inline' }
+      end
+
+      def raw_redirect_params
+        { query: { 'response-content-type' => 'text/plain; charset=utf-8', 'response-content-disposition' => 'inline' } }
+      end
+
+      def trace_artifact_file
+        @trace_artifact_file ||= build.job_artifacts_trace&.file
+      end
+    end
+  end
+end

ee/app/models/ee/ci/job_artifact.rb

@@ -9,11 +9,11 @@ module EE
     prepended do
       after_destroy :log_geo_event
 
-      scope :with_files_stored_locally, -> { where(file_store: [nil, JobArtifactUploader::Store::LOCAL]) }
+      scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
     end
 
     def local_store?
-      [nil, JobArtifactUploader::Store::LOCAL].include?(self.file_store)
+      [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
     end
 
     private

ee/app/uploaders/ee/job_artifact_uploader.rb

+module EE
+  module JobArtifactUploader
+    extend ActiveSupport::Concern
+
+    def open
+      if file_storage?
+        super
+      else
+        ::Gitlab::Ci::Trace::HttpIO.new(url, size) if url
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/trace/http_io.rb

+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  module Ci
+    class Trace
+      class HttpIO
+        BUFFER_SIZE = 128.kilobytes
+
+        InvalidURLError = Class.new(StandardError)
+        FailedToGetChunkError = Class.new(StandardError)
+
+        attr_reader :uri, :size
+        attr_reader :tell
+        attr_reader :chunk, :chunk_range
+
+        alias_method :pos, :tell
+
+        def initialize(url, size)
+          raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url)
+
+          @uri = URI(url)
+          @size = size
+          @tell = 0
+        end
+
+        def close
+          # no-op
+        end
+
+        def binmode
+          # no-op
+        end
+
+        def binmode?
+          true
+        end
+
+        def path
+          @uri.to_s
+        end
+
+        def seek(pos, where = IO::SEEK_SET)
+          new_pos =
+            case where
+            when IO::SEEK_END
+              size + pos
+            when IO::SEEK_SET
+              pos
+            when IO::SEEK_CUR
+              tell + pos
+            else
+              -1
+            end
+
+          raise 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+          @tell = new_pos
+        end
+
+        def eof?
+          tell == size
+        end
+
+        def each_line
+          until eof?
+            line = readline
+            break if line.nil?
+
+            yield(line)
+          end
+        end
+
+        def read(length = nil)
+          out = ""
+
+          until eof? || (length && out.length >= length)
+            data = get_chunk
+            break if data.empty?
+
+            out << data
+            @tell += data.bytesize
+          end
+
+          out = out[0, length] if length && out.length > length
+
+          out
+        end
+
+        def readline
+          out = ""
+
+          until eof?
+            data = get_chunk
+            new_line = data.index("\n")
+
+            if !new_line.nil?
+              out << data[0..new_line]
+              @tell += new_line + 1
+              break
+            else
+              out << data
+              @tell += data.bytesize
+            end
+          end
+
+          out
+        end
+
+        def write(data)
+          raise NotImplementedError
+        end
+
+        def truncate(offset)
+          raise NotImplementedError
+        end
+
+        def flush
+          raise NotImplementedError
+        end
+
+        def present?
+          true
+        end
+
+        private
+
+        ##
+        # The below methods are not implemented in IO class
+        #
+        def in_range?
+          @chunk_range&.include?(tell)
+        end
+
+        def get_chunk
+          unless in_range?
+            response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+              http.request(request)
+            end
+
+            raise FailedToGetChunkError unless response.code == '200' || response.code == '206'
+
+            @chunk = response.body.force_encoding(Encoding::BINARY)
+            @chunk_range = response.content_range
+
+            ##
+            # Note: If provider does not return content_range, then we set it as we requested
+            # Provider: minio
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: AWS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: GCS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200
+            @chunk_range ||= (chunk_start...(chunk_start + @chunk.length))
+          end
+
+          @chunk[chunk_offset..BUFFER_SIZE]
+        end
+
+        def request
+          Net::HTTP::Get.new(uri).tap do |request|
+            request.set_range(chunk_start, BUFFER_SIZE)
+          end
+        end
+
+        def chunk_offset
+          tell % BUFFER_SIZE
+        end
+
+        def chunk_start
+          (tell / BUFFER_SIZE) * BUFFER_SIZE
+        end
+
+        def chunk_end
+          [chunk_start + BUFFER_SIZE, size].min
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/trace.rb

@@ -52,12 +52,14 @@ module Gitlab
       end
 
       def exist?
-        current_path.present? || old_trace.present?
+        trace_artifact&.exists? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
-          if current_path
+          if trace_artifact
+            trace_artifact.open
+          elsif current_path
             File.open(current_path, "rb")
           elsif old_trace
             StringIO.new(old_trace)
@@ -82,6 +84,8 @@ module Gitlab
       end
 
       def erase!
+        trace_artifact&.destroy
+
         paths.each do |trace_path|
           FileUtils.rm(trace_path, force: true)
         end
@@ -137,6 +141,10 @@ module Gitlab
           "#{job.id}.log"
         ) if job.project&.ci_id
       end
+
+      def trace_artifact
+        job.job_artifacts_trace
+      end
     end
   end
 end

spec/controllers/projects/jobs_controller_spec.rb

@@ -159,8 +159,19 @@ describe Projects::JobsController do
       get_trace
     end
 
+    context 'when job has a trace artifact' do
+      let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+      it 'returns a trace' do
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response['id']).to eq job.id
+        expect(json_response['status']).to eq job.status
+        expect(json_response['html']).to eq(job.trace.html)
+      end
+    end
+
     context 'when job has a trace' do
-      let(:job) { create(:ci_build, :trace, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
 
       it 'returns a trace' do
         expect(response).to have_gitlab_http_status(:ok)
@@ -182,7 +193,7 @@ describe Projects::JobsController do
     end
 
     context 'when job has a trace with ANSI sequence and Unicode' do
-      let(:job) { create(:ci_build, :unicode_trace, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :unicode_trace_live, pipeline: pipeline) }
 
       it 'returns a trace with Unicode' do
         expect(response).to have_gitlab_http_status(:ok)
@@ -381,7 +392,7 @@ describe Projects::JobsController do
     end
 
     context 'when job is erasable' do
-      let(:job) { create(:ci_build, :erasable, :trace, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :erasable, :trace_artifact, pipeline: pipeline) }
 
       it 'redirects to the erased job page' do
         expect(response).to have_gitlab_http_status(:found)
@@ -408,7 +419,7 @@ describe Projects::JobsController do
 
     context 'when user is developer' do
       let(:role) { :developer }
-      let(:job) { create(:ci_build, :erasable, :trace, pipeline: pipeline, user: triggered_by) }
+      let(:job) { create(:ci_build, :erasable, :trace_artifact, pipeline: pipeline, user: triggered_by) }
 
       context 'when triggered by same user' do
         let(:triggered_by) { user }
@@ -439,8 +450,18 @@ describe Projects::JobsController do
       get_raw
     end
 
+    context 'when job has a trace artifact' do
+      let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+      it 'returns a trace' do
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response.content_type).to eq 'text/plain; charset=utf-8'
+        expect(response.body).to eq job.job_artifacts_trace.open.read
+      end
+    end
+
     context 'when job has a trace file' do
-      let(:job) { create(:ci_build, :trace, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
 
       it 'send a trace file' do
         expect(response).to have_gitlab_http_status(:ok)

spec/ee/spec/controllers/ee/projects/jobs_controller_spec.rb

+require 'spec_helper'
+
+describe Projects::JobsController do
+  include ApiHelpers
+  include HttpIOHelpers
+
+  let(:project) { create(:project, :public) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+
+  describe 'GET trace.json' do
+    context 'when trace artifact is in ObjectStorage' do
+      let!(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) }
+
+      before do
+        allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+        allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+        allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+      end
+
+      context 'when there are no network issues' do
+        before do
+          stub_remote_trace_206
+
+          get_trace
+        end
+
+        it 'returns a trace' do
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response['id']).to eq job.id
+          expect(json_response['status']).to eq job.status
+          expect(json_response['html']).to eq(job.trace.html)
+        end
+      end
+
+      context 'when there is a network issue' do
+        before do
+          stub_remote_trace_500
+        end
+
+        it 'returns a trace' do
+          expect { get_trace }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+        end
+      end
+    end
+
+    def get_trace
+      get :trace, namespace_id: project.namespace,
+                  project_id: project,
+                  id: job.id,
+                  format: :json
+    end
+  end
+
+  describe 'GET raw' do
+    subject do
+      post :raw, namespace_id: project.namespace,
+                 project_id: project,
+                 id: job.id
+    end
+
+    context 'when the trace artifact is in ObjectStorage' do
+      let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+      before do
+        allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+      end
+
+      it 'redirect to the trace file url' do
+        expect(subject).to redirect_to(job.job_artifacts_trace.file.url)
+      end
+    end
+  end
+end

spec/ee/spec/lib/gitlab/ci/trace/http_io_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::HttpIO do
+  include HttpIOHelpers
+
+  let(:http_io) { described_class.new(url, size) }
+  let(:url) { remote_trace_url }
+  let(:size) { remote_trace_size }
+
+  describe 'Interchangeability between IO and HttpIO' do
+    EXCEPT_METHODS = %i[read_nonblock raw raw! cooked cooked! getch echo= echo?
+                        winsize winsize= iflush oflush ioflush beep goto cursor cursor= pressed?
+                        getpass write_nonblock stat pathconf wait_readable wait_writable getbyte <<
+                        wait lines bytes chars codepoints getc readpartial set_encoding printf print
+                        putc puts readlines gets each each_byte each_char each_codepoint to_io reopen
+                        syswrite to_i fileno sysread fdatasync fsync sync= sync lineno= lineno readchar
+                        ungetbyte readbyte ungetc nonblock= nread rewind pos= eof close_on_exec?
+                        close_on_exec= closed? close_read close_write isatty tty? binmode? sysseek
+                        advise ioctl fcntl pid external_encoding internal_encoding autoclose? autoclose=
+                        posix_fileno nonblock? ready? noecho nonblock].freeze
+
+    it 'HttpIO covers core interfaces in IO' do
+      expected_interfaces = ::IO.instance_methods(false)
+      expected_interfaces -= EXCEPT_METHODS
+
+      expect(expected_interfaces - described_class.instance_methods).to be_empty
+    end
+  end
+
+  describe '#close' do
+    subject { http_io.close }
+
+    it { is_expected.to be_nil }
+  end
+
+  describe '#binmode' do
+    subject { http_io.binmode }
+
+    it { is_expected.to be_nil }
+  end
+
+  describe '#binmode?' do
+    subject { http_io.binmode? }
+
+    it { is_expected.to be_truthy }
+  end
+
+  describe '#path' do
+    subject { http_io.path }
+
+    it { is_expected.to eq(url) }
+  end
+
+  describe '#seek' do
+    subject { http_io.seek(pos, where) }
+
+    context 'when moves pos to end of the file' do
+      let(:pos) { 0 }
+      let(:where) { IO::SEEK_END }
+
+      it { is_expected.to eq(size) }
+    end
+
+    context 'when moves pos to middle of the file' do
+      let(:pos) { size / 2 }
+      let(:where) { IO::SEEK_SET }
+
+      it { is_expected.to eq(size / 2) }
+    end
+
+    context 'when moves pos around' do
+      it 'matches the result' do
+        expect(http_io.seek(0)).to eq(0)
+        expect(http_io.seek(100, IO::SEEK_CUR)).to eq(100)
+        expect { http_io.seek(size + 1, IO::SEEK_CUR) }.to raise_error('new position is outside of file')
+      end
+    end
+  end
+
+  describe '#eof?' do
+    subject { http_io.eof? }
+
+    context 'when current pos is at end of the file' do
+      before do
+        http_io.seek(size, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when current pos is not at end of the file' do
+      before do
+        http_io.seek(0, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#each_line' do
+    subject { http_io.each_line }
+
+    let(:string_io) { StringIO.new(remote_trace_body) }
+
+    before do
+      stub_remote_trace_206
+    end
+
+    it 'yields lines' do
+      expect { |b| http_io.each_line(&b) }.to yield_successive_args(*string_io.each_line.to_a)
+    end
+
+    context 'when buckets on GCS' do
+      context 'when BUFFER_SIZE is larger than file size' do
+        before do
+          stub_remote_trace_200
+          set_larger_buffer_size_than(size)
+        end
+
+        it 'calls get_chunk only once' do
+          expect_any_instance_of(Net::HTTP).to receive(:request).once.and_call_original
+
+          http_io.each_line { |line| }
+        end
+      end
+    end
+  end
+
+  describe '#read' do
+    subject { http_io.read(length) }
+
+    context 'when there are no network issue' do
+      before do
+        stub_remote_trace_206
+      end
+
+      context 'when read whole size' do
+        let(:length) { nil }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+      end
+
+      context 'when read only first 100 bytes' do
+        let(:length) { 100 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body[0, length])
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body[0, length])
+          end
+        end
+      end
+
+      context 'when tries to read oversize' do
+        let(:length) { size + 1000 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to eq(remote_trace_body)
+          end
+        end
+      end
+
+      context 'when tries to read 0 bytes' do
+        let(:length) { 0 }
+
+        context 'when BUFFER_SIZE is smaller than file size' do
+          before do
+            set_smaller_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to be_empty
+          end
+        end
+
+        context 'when BUFFER_SIZE is larger than file size' do
+          before do
+            set_larger_buffer_size_than(size)
+          end
+
+          it 'reads a trace' do
+            is_expected.to be_empty
+          end
+        end
+      end
+    end
+
+    context 'when there is anetwork issue' do
+      let(:length) { nil }
+
+      before do
+        stub_remote_trace_500
+      end
+
+      it 'reads a trace' do
+        expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+      end
+    end
+  end
+
+  describe '#readline' do
+    subject { http_io.readline }
+
+    let(:string_io) { StringIO.new(remote_trace_body) }
+
+    before do
+      stub_remote_trace_206
+    end
+
+    shared_examples 'all line matching' do
+      it 'reads a line' do
+        (0...remote_trace_body.lines.count).each do
+          expect(http_io.readline).to eq(string_io.readline)
+        end
+      end
+    end
+
+    context 'when there is anetwork issue' do
+      let(:length) { nil }
+
+      before do
+        stub_remote_trace_500
+      end
+
+      it 'reads a trace' do
+        expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError)
+      end
+    end
+
+    context 'when BUFFER_SIZE is smaller than file size' do
+      before do
+        set_smaller_buffer_size_than(size)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when BUFFER_SIZE is larger than file size' do
+      before do
+        set_larger_buffer_size_than(size)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when pos is at middle of the file' do
+      before do
+        set_smaller_buffer_size_than(size)
+
+        http_io.seek(size / 2)
+        string_io.seek(size / 2)
+      end
+
+      it 'reads from pos' do
+        expect(http_io.readline).to eq(string_io.readline)
+      end
+    end
+  end
+
+  describe '#write' do
+    subject { http_io.write(nil) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#truncate' do
+    subject { http_io.truncate(nil) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#flush' do
+    subject { http_io.flush }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#present?' do
+    subject { http_io.present? }
+
+    it { is_expected.to be_truthy }
+  end
+end

spec/ee/spec/uploaders/ee/job_artifact_uploader_spec.rb

+require 'spec_helper'
+
+describe JobArtifactUploader do
+  let(:store) { ObjectStorage::Store::LOCAL }
+  let(:job_artifact) { create(:ci_job_artifact, file_store: store) }
+  let(:uploader) { described_class.new(job_artifact, :file) }
+
+  describe '#open' do
+    subject { uploader.open }
+
+    context 'when trace is stored in Object storage' do
+      before do
+        allow(uploader).to receive(:file_storage?) { false }
+        allow(uploader).to receive(:url) { 'http://object_storage.com/trace' }
+      end
+
+      it 'returns http io stream' do
+        is_expected.to be_a(Gitlab::Ci::Trace::HttpIO)
+      end
+    end
+  end
+end

spec/factories/ci/builds.rb

@@ -135,13 +135,19 @@ FactoryBot.define do
       coverage_regex '/(d+)/'
     end
 
-    trait :trace do
+    trait :trace_live do
       after(:create) do |build, evaluator|
         build.trace.set('BUILD TRACE')
       end
     end
 
-    trait :unicode_trace do
+    trait :trace_artifact do
+      after(:create) do |build, evaluator|
+        create(:ci_job_artifact, :trace, job: build)
+      end
+    end
+
+    trait :unicode_trace_live do
       after(:create) do |build, evaluator|
         trace = File.binread(
           File.expand_path(

spec/factories/ci/job_artifacts.rb

@@ -30,5 +30,14 @@ FactoryBot.define do
           Rails.root.join('spec/fixtures/ci_build_artifacts_metadata.gz'), 'application/x-gzip')
       end
     end
+
+    trait :trace do
+      file_type :trace
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
+      end
+    end
   end
 end

spec/features/projects/jobs_spec.rb

@@ -7,7 +7,7 @@ feature 'Jobs' do
   let(:project) { create(:project, :repository) }
   let(:pipeline) { create(:ci_pipeline, project: project) }
 
-  let(:job) { create(:ci_build, :trace, pipeline: pipeline) }
+  let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
   let(:job2) { create(:ci_build) }
 
   let(:artifacts_file) do
@@ -468,18 +468,34 @@ feature 'Jobs' do
   describe 'GET /:project/jobs/:id/raw', :js do
     context 'access source' do
       context 'job from project' do
-        before do
-          job.run!
-        end
+        context 'when job is running' do
+          before do
+            job.run!
+          end
 
-        it 'sends the right headers' do
-          requests = inspect_requests(inject_headers: { 'X-Sendfile-Type' => 'X-Sendfile' }) do
-            visit raw_project_job_path(project, job)
+          it 'sends the right headers' do
+            requests = inspect_requests(inject_headers: { 'X-Sendfile-Type' => 'X-Sendfile' }) do
+              visit raw_project_job_path(project, job)
+            end
+
+            expect(requests.first.status_code).to eq(200)
+            expect(requests.first.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
+            expect(requests.first.response_headers['X-Sendfile']).to eq(job.trace.send(:current_path))
           end
+        end
 
-          expect(requests.first.status_code).to eq(200)
-          expect(requests.first.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
-          expect(requests.first.response_headers['X-Sendfile']).to eq(job.trace.send(:current_path))
+        context 'when job is complete' do
+          let(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) }
+
+          it 'sends the right headers' do
+            requests = inspect_requests(inject_headers: { 'X-Sendfile-Type' => 'X-Sendfile' }) do
+              visit raw_project_job_path(project, job)
+            end
+
+            expect(requests.first.status_code).to eq(200)
+            expect(requests.first.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
+            expect(requests.first.response_headers['X-Sendfile']).to eq(job.job_artifacts_trace.file.path)
+          end
         end
       end
 

spec/fixtures/trace/sample_trace

+Running with gitlab-runner 10.4.0 (857480b6)
+  on docker-auto-scale-com (9a6801bd)
+Using Docker executor with image dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 ...
+Starting service postgres:9.2 ...
+Pulling docker image postgres:9.2 ...
+Using docker image postgres:9.2 ID=sha256:18cdbca56093c841d28e629eb8acd4224afe0aa4c57c839351fc181888b8a470 for postgres service...
+Starting service redis:alpine ...
+Pulling docker image redis:alpine ...
+Using docker image redis:alpine ID=sha256:cb1ec54b370d4a91dff57d00f91fd880dc710160a58440adaa133e0f84ae999d for redis service...
+Waiting for services to be up and running...
+Using docker image sha256:3006a02a5a6f0a116358a13bbc46ee46fb2471175efd5b7f9b1c22345ec2a8e9 for predefined container...
+Pulling docker image dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 ...
+Using docker image dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 ID=sha256:1f59be408f12738509ffe4177d65e9de6391f32461de83d9d45f58517b30af99 for build container...
+section_start:1517486886:prepare_script
+Running on runner-9a6801bd-project-13083-concurrent-0 via runner-9a6801bd-gsrm-1517484168-a8449153...
+section_end:1517486887:prepare_script
+section_start:1517486887:get_sources
+Fetching changes for 42624-gitaly-bundle-isolation-not-working-in-ci with git depth set to 20...
+Removing .gitlab_shell_secret
+Removing .gitlab_workhorse_secret
+Removing .yarn-cache/
+Removing config/database.yml
+Removing config/gitlab.yml
+Removing config/redis.cache.yml
+Removing config/redis.queues.yml
+Removing config/redis.shared_state.yml
+Removing config/resque.yml
+Removing config/secrets.yml
+Removing coverage/
+Removing knapsack/
+Removing log/api_json.log
+Removing log/application.log
+Removing log/gitaly-test.log
+Removing log/githost.log
+Removing log/grpc.log
+Removing log/test_json.log
+Removing node_modules/
+Removing public/assets/
+Removing rspec_flaky/
+Removing shared/tmp/
+Removing tmp/tests/
+Removing vendor/ruby/
+HEAD is now at 4cea24f Converted todos.js to axios
+From https://gitlab.com/gitlab-org/gitlab-ce
+ * [new branch]      42624-gitaly-bundle-isolation-not-working-in-ci -> origin/42624-gitaly-bundle-isolation-not-working-in-ci
+Checking out f42a5e24 as 42624-gitaly-bundle-isolation-not-working-in-ci...
+Skipping Git submodules setup
+section_end:1517486896:get_sources
+section_start:1517486896:restore_cache
+Checking cache for ruby-2.3.6-with-yarn...
+Downloading cache.zip from http://runners-cache-5-internal.gitlab.com:444/runner/project/13083/ruby-2.3.6-with-yarn 
+Successfully extracted cache
+section_end:1517486919:restore_cache
+section_start:1517486919:download_artifacts
+Downloading artifacts for retrieve-tests-metadata (50551658)...
+Downloading artifacts from coordinator... ok        id=50551658 responseStatus=200 OK token=HhF7y_1X
+Downloading artifacts for compile-assets (50551659)...
+Downloading artifacts from coordinator... ok        id=50551659 responseStatus=200 OK token=wTz6JrCP
+Downloading artifacts for setup-test-env (50551660)...
+Downloading artifacts from coordinator... ok        id=50551660 responseStatus=200 OK token=DTGgeVF5
+WARNING: tmp/tests/gitlab-shell/.gitlab_shell_secret: chmod tmp/tests/gitlab-shell/.gitlab_shell_secret: no such file or directory (suppressing repeats) 
+section_end:1517486934:download_artifacts
+section_start:1517486934:build_script
+$ bundle --version
+Bundler version 1.16.1
+$ source scripts/utils.sh
+$ source scripts/prepare_build.sh
+The Gemfile's dependencies are satisfied
+Successfully installed knapsack-1.15.0
+1 gem installed
+NOTICE:  database "gitlabhq_test" does not exist, skipping
+DROP DATABASE
+CREATE DATABASE
+CREATE ROLE
+GRANT
+-- enable_extension("plpgsql")
+   -> 0.0156s
+-- enable_extension("pg_trgm")
+   -> 0.0156s
+-- create_table("abuse_reports", {:force=>:cascade})
+   -> 0.0119s
+-- create_table("appearances", {:force=>:cascade})
+   -> 0.0065s
+-- create_table("application_settings", {:force=>:cascade})
+   -> 0.0382s
+-- create_table("audit_events", {:force=>:cascade})
+   -> 0.0056s
+-- add_index("audit_events", ["entity_id", "entity_type"], {:name=>"index_audit_events_on_entity_id_and_entity_type", :using=>:btree})
+   -> 0.0040s
+-- create_table("award_emoji", {:force=>:cascade})
+   -> 0.0058s
+-- add_index("award_emoji", ["awardable_type", "awardable_id"], {:name=>"index_award_emoji_on_awardable_type_and_awardable_id", :using=>:btree})
+   -> 0.0068s
+-- add_index("award_emoji", ["user_id", "name"], {:name=>"index_award_emoji_on_user_id_and_name", :using=>:btree})
+   -> 0.0043s
+-- create_table("boards", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("boards", ["project_id"], {:name=>"index_boards_on_project_id", :using=>:btree})
+   -> 0.0056s
+-- create_table("broadcast_messages", {:force=>:cascade})
+   -> 0.0056s
+-- add_index("broadcast_messages", ["starts_at", "ends_at", "id"], {:name=>"index_broadcast_messages_on_starts_at_and_ends_at_and_id", :using=>:btree})
+   -> 0.0041s
+-- create_table("chat_names", {:force=>:cascade})
+   -> 0.0056s
+-- add_index("chat_names", ["service_id", "team_id", "chat_id"], {:name=>"index_chat_names_on_service_id_and_team_id_and_chat_id", :unique=>true, :using=>:btree})
+   -> 0.0039s
+-- add_index("chat_names", ["user_id", "service_id"], {:name=>"index_chat_names_on_user_id_and_service_id", :unique=>true, :using=>:btree})
+   -> 0.0036s
+-- create_table("chat_teams", {:force=>:cascade})
+   -> 0.0068s
+-- add_index("chat_teams", ["namespace_id"], {:name=>"index_chat_teams_on_namespace_id", :unique=>true, :using=>:btree})
+   -> 0.0098s
+-- create_table("ci_build_trace_section_names", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("ci_build_trace_section_names", ["project_id", "name"], {:name=>"index_ci_build_trace_section_names_on_project_id_and_name", :unique=>true, :using=>:btree})
+   -> 0.0035s
+-- create_table("ci_build_trace_sections", {:force=>:cascade})
+   -> 0.0040s
+-- add_index("ci_build_trace_sections", ["build_id", "section_name_id"], {:name=>"index_ci_build_trace_sections_on_build_id_and_section_name_id", :unique=>true, :using=>:btree})
+   -> 0.0035s
+-- add_index("ci_build_trace_sections", ["project_id"], {:name=>"index_ci_build_trace_sections_on_project_id", :using=>:btree})
+   -> 0.0033s
+-- create_table("ci_builds", {:force=>:cascade})
+   -> 0.0062s
+-- add_index("ci_builds", ["auto_canceled_by_id"], {:name=>"index_ci_builds_on_auto_canceled_by_id", :using=>:btree})
+   -> 0.0035s
+-- add_index("ci_builds", ["commit_id", "stage_idx", "created_at"], {:name=>"index_ci_builds_on_commit_id_and_stage_idx_and_created_at", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_builds", ["commit_id", "status", "type"], {:name=>"index_ci_builds_on_commit_id_and_status_and_type", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_builds", ["commit_id", "type", "name", "ref"], {:name=>"index_ci_builds_on_commit_id_and_type_and_name_and_ref", :using=>:btree})
+   -> 0.0035s
+-- add_index("ci_builds", ["commit_id", "type", "ref"], {:name=>"index_ci_builds_on_commit_id_and_type_and_ref", :using=>:btree})
+   -> 0.0042s
+-- add_index("ci_builds", ["project_id", "id"], {:name=>"index_ci_builds_on_project_id_and_id", :using=>:btree})
+   -> 0.0031s
+-- add_index("ci_builds", ["protected"], {:name=>"index_ci_builds_on_protected", :using=>:btree})
+   -> 0.0031s
+-- add_index("ci_builds", ["runner_id"], {:name=>"index_ci_builds_on_runner_id", :using=>:btree})
+   -> 0.0033s
+-- add_index("ci_builds", ["stage_id"], {:name=>"index_ci_builds_on_stage_id", :using=>:btree})
+   -> 0.0035s
+-- add_index("ci_builds", ["status", "type", "runner_id"], {:name=>"index_ci_builds_on_status_and_type_and_runner_id", :using=>:btree})
+   -> 0.0031s
+-- add_index("ci_builds", ["status"], {:name=>"index_ci_builds_on_status", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_builds", ["token"], {:name=>"index_ci_builds_on_token", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("ci_builds", ["updated_at"], {:name=>"index_ci_builds_on_updated_at", :using=>:btree})
+   -> 0.0047s
+-- add_index("ci_builds", ["user_id"], {:name=>"index_ci_builds_on_user_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("ci_group_variables", {:force=>:cascade})
+   -> 0.0055s
+-- add_index("ci_group_variables", ["group_id", "key"], {:name=>"index_ci_group_variables_on_group_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- create_table("ci_job_artifacts", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("ci_job_artifacts", ["job_id", "file_type"], {:name=>"index_ci_job_artifacts_on_job_id_and_file_type", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- add_index("ci_job_artifacts", ["project_id"], {:name=>"index_ci_job_artifacts_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("ci_pipeline_schedule_variables", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("ci_pipeline_schedule_variables", ["pipeline_schedule_id", "key"], {:name=>"index_ci_pipeline_schedule_variables_on_schedule_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- create_table("ci_pipeline_schedules", {:force=>:cascade})
+   -> 0.0047s
+-- add_index("ci_pipeline_schedules", ["next_run_at", "active"], {:name=>"index_ci_pipeline_schedules_on_next_run_at_and_active", :using=>:btree})
+   -> 0.0029s
+-- add_index("ci_pipeline_schedules", ["project_id"], {:name=>"index_ci_pipeline_schedules_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("ci_pipeline_variables", {:force=>:cascade})
+   -> 0.0045s
+-- add_index("ci_pipeline_variables", ["pipeline_id", "key"], {:name=>"index_ci_pipeline_variables_on_pipeline_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- create_table("ci_pipelines", {:force=>:cascade})
+   -> 0.0057s
+-- add_index("ci_pipelines", ["auto_canceled_by_id"], {:name=>"index_ci_pipelines_on_auto_canceled_by_id", :using=>:btree})
+   -> 0.0030s
+-- add_index("ci_pipelines", ["pipeline_schedule_id"], {:name=>"index_ci_pipelines_on_pipeline_schedule_id", :using=>:btree})
+   -> 0.0031s
+-- add_index("ci_pipelines", ["project_id", "ref", "status", "id"], {:name=>"index_ci_pipelines_on_project_id_and_ref_and_status_and_id", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_pipelines", ["project_id", "sha"], {:name=>"index_ci_pipelines_on_project_id_and_sha", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_pipelines", ["project_id"], {:name=>"index_ci_pipelines_on_project_id", :using=>:btree})
+   -> 0.0035s
+-- add_index("ci_pipelines", ["status"], {:name=>"index_ci_pipelines_on_status", :using=>:btree})
+   -> 0.0032s
+-- add_index("ci_pipelines", ["user_id"], {:name=>"index_ci_pipelines_on_user_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("ci_runner_projects", {:force=>:cascade})
+   -> 0.0035s
+-- add_index("ci_runner_projects", ["project_id"], {:name=>"index_ci_runner_projects_on_project_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("ci_runner_projects", ["runner_id"], {:name=>"index_ci_runner_projects_on_runner_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("ci_runners", {:force=>:cascade})
+   -> 0.0059s
+-- add_index("ci_runners", ["contacted_at"], {:name=>"index_ci_runners_on_contacted_at", :using=>:btree})
+   -> 0.0030s
+-- add_index("ci_runners", ["is_shared"], {:name=>"index_ci_runners_on_is_shared", :using=>:btree})
+   -> 0.0030s
+-- add_index("ci_runners", ["locked"], {:name=>"index_ci_runners_on_locked", :using=>:btree})
+   -> 0.0030s
+-- add_index("ci_runners", ["token"], {:name=>"index_ci_runners_on_token", :using=>:btree})
+   -> 0.0029s
+-- create_table("ci_stages", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("ci_stages", ["pipeline_id", "name"], {:name=>"index_ci_stages_on_pipeline_id_and_name", :using=>:btree})
+   -> 0.0031s
+-- add_index("ci_stages", ["pipeline_id"], {:name=>"index_ci_stages_on_pipeline_id", :using=>:btree})
+   -> 0.0030s
+-- add_index("ci_stages", ["project_id"], {:name=>"index_ci_stages_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("ci_trigger_requests", {:force=>:cascade})
+   -> 0.0058s
+-- add_index("ci_trigger_requests", ["commit_id"], {:name=>"index_ci_trigger_requests_on_commit_id", :using=>:btree})
+   -> 0.0031s
+-- create_table("ci_triggers", {:force=>:cascade})
+   -> 0.0043s
+-- add_index("ci_triggers", ["project_id"], {:name=>"index_ci_triggers_on_project_id", :using=>:btree})
+   -> 0.0033s
+-- create_table("ci_variables", {:force=>:cascade})
+   -> 0.0059s
+-- add_index("ci_variables", ["project_id", "key", "environment_scope"], {:name=>"index_ci_variables_on_project_id_and_key_and_environment_scope", :unique=>true, :using=>:btree})
+   -> 0.0031s
+-- create_table("cluster_platforms_kubernetes", {:force=>:cascade})
+   -> 0.0053s
+-- add_index("cluster_platforms_kubernetes", ["cluster_id"], {:name=>"index_cluster_platforms_kubernetes_on_cluster_id", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- create_table("cluster_projects", {:force=>:cascade})
+   -> 0.0032s
+-- add_index("cluster_projects", ["cluster_id"], {:name=>"index_cluster_projects_on_cluster_id", :using=>:btree})
+   -> 0.0035s
+-- add_index("cluster_projects", ["project_id"], {:name=>"index_cluster_projects_on_project_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("cluster_providers_gcp", {:force=>:cascade})
+   -> 0.0051s
+-- add_index("cluster_providers_gcp", ["cluster_id"], {:name=>"index_cluster_providers_gcp_on_cluster_id", :unique=>true, :using=>:btree})
+   -> 0.0034s
+-- create_table("clusters", {:force=>:cascade})
+   -> 0.0052s
+-- add_index("clusters", ["enabled"], {:name=>"index_clusters_on_enabled", :using=>:btree})
+   -> 0.0031s
+-- add_index("clusters", ["user_id"], {:name=>"index_clusters_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("clusters_applications_helm", {:force=>:cascade})
+   -> 0.0045s
+-- create_table("clusters_applications_ingress", {:force=>:cascade})
+   -> 0.0044s
+-- create_table("clusters_applications_prometheus", {:force=>:cascade})
+   -> 0.0047s
+-- create_table("container_repositories", {:force=>:cascade})
+   -> 0.0050s
+-- add_index("container_repositories", ["project_id", "name"], {:name=>"index_container_repositories_on_project_id_and_name", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("container_repositories", ["project_id"], {:name=>"index_container_repositories_on_project_id", :using=>:btree})
+   -> 0.0032s
+-- create_table("conversational_development_index_metrics", {:force=>:cascade})
+   -> 0.0076s
+-- create_table("deploy_keys_projects", {:force=>:cascade})
+   -> 0.0037s
+-- add_index("deploy_keys_projects", ["project_id"], {:name=>"index_deploy_keys_projects_on_project_id", :using=>:btree})
+   -> 0.0032s
+-- create_table("deployments", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("deployments", ["created_at"], {:name=>"index_deployments_on_created_at", :using=>:btree})
+   -> 0.0034s
+-- add_index("deployments", ["environment_id", "id"], {:name=>"index_deployments_on_environment_id_and_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("deployments", ["environment_id", "iid", "project_id"], {:name=>"index_deployments_on_environment_id_and_iid_and_project_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("deployments", ["project_id", "iid"], {:name=>"index_deployments_on_project_id_and_iid", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- create_table("emails", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("emails", ["confirmation_token"], {:name=>"index_emails_on_confirmation_token", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("emails", ["email"], {:name=>"index_emails_on_email", :unique=>true, :using=>:btree})
+   -> 0.0035s
+-- add_index("emails", ["user_id"], {:name=>"index_emails_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("environments", {:force=>:cascade})
+   -> 0.0052s
+-- add_index("environments", ["project_id", "name"], {:name=>"index_environments_on_project_id_and_name", :unique=>true, :using=>:btree})
+   -> 0.0031s
+-- add_index("environments", ["project_id", "slug"], {:name=>"index_environments_on_project_id_and_slug", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- create_table("events", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("events", ["action"], {:name=>"index_events_on_action", :using=>:btree})
+   -> 0.0032s
+-- add_index("events", ["author_id"], {:name=>"index_events_on_author_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("events", ["project_id", "id"], {:name=>"index_events_on_project_id_and_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("events", ["target_type", "target_id"], {:name=>"index_events_on_target_type_and_target_id", :using=>:btree})
+   -> 0.0027s
+-- create_table("feature_gates", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("feature_gates", ["feature_key", "key", "value"], {:name=>"index_feature_gates_on_feature_key_and_key_and_value", :unique=>true, :using=>:btree})
+   -> 0.0031s
+-- create_table("features", {:force=>:cascade})
+   -> 0.0041s
+-- add_index("features", ["key"], {:name=>"index_features_on_key", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- create_table("fork_network_members", {:force=>:cascade})
+   -> 0.0033s
+-- add_index("fork_network_members", ["fork_network_id"], {:name=>"index_fork_network_members_on_fork_network_id", :using=>:btree})
+   -> 0.0033s
+-- add_index("fork_network_members", ["project_id"], {:name=>"index_fork_network_members_on_project_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("fork_networks", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("fork_networks", ["root_project_id"], {:name=>"index_fork_networks_on_root_project_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("forked_project_links", {:force=>:cascade})
+   -> 0.0032s
+-- add_index("forked_project_links", ["forked_to_project_id"], {:name=>"index_forked_project_links_on_forked_to_project_id", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- create_table("gcp_clusters", {:force=>:cascade})
+   -> 0.0074s
+-- add_index("gcp_clusters", ["project_id"], {:name=>"index_gcp_clusters_on_project_id", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- create_table("gpg_key_subkeys", {:force=>:cascade})
+   -> 0.0042s
+-- add_index("gpg_key_subkeys", ["fingerprint"], {:name=>"index_gpg_key_subkeys_on_fingerprint", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- add_index("gpg_key_subkeys", ["gpg_key_id"], {:name=>"index_gpg_key_subkeys_on_gpg_key_id", :using=>:btree})
+   -> 0.0032s
+-- add_index("gpg_key_subkeys", ["keyid"], {:name=>"index_gpg_key_subkeys_on_keyid", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- create_table("gpg_keys", {:force=>:cascade})
+   -> 0.0042s
+-- add_index("gpg_keys", ["fingerprint"], {:name=>"index_gpg_keys_on_fingerprint", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("gpg_keys", ["primary_keyid"], {:name=>"index_gpg_keys_on_primary_keyid", :unique=>true, :using=>:btree})
+   -> 0.0026s
+-- add_index("gpg_keys", ["user_id"], {:name=>"index_gpg_keys_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("gpg_signatures", {:force=>:cascade})
+   -> 0.0054s
+-- add_index("gpg_signatures", ["commit_sha"], {:name=>"index_gpg_signatures_on_commit_sha", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- add_index("gpg_signatures", ["gpg_key_id"], {:name=>"index_gpg_signatures_on_gpg_key_id", :using=>:btree})
+   -> 0.0026s
+-- add_index("gpg_signatures", ["gpg_key_primary_keyid"], {:name=>"index_gpg_signatures_on_gpg_key_primary_keyid", :using=>:btree})
+   -> 0.0029s
+-- add_index("gpg_signatures", ["gpg_key_subkey_id"], {:name=>"index_gpg_signatures_on_gpg_key_subkey_id", :using=>:btree})
+   -> 0.0032s
+-- add_index("gpg_signatures", ["project_id"], {:name=>"index_gpg_signatures_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("group_custom_attributes", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("group_custom_attributes", ["group_id", "key"], {:name=>"index_group_custom_attributes_on_group_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("group_custom_attributes", ["key", "value"], {:name=>"index_group_custom_attributes_on_key_and_value", :using=>:btree})
+   -> 0.0028s
+-- create_table("identities", {:force=>:cascade})
+   -> 0.0043s
+-- add_index("identities", ["user_id"], {:name=>"index_identities_on_user_id", :using=>:btree})
+   -> 0.0034s
+-- create_table("issue_assignees", {:id=>false, :force=>:cascade})
+   -> 0.0013s
+-- add_index("issue_assignees", ["issue_id", "user_id"], {:name=>"index_issue_assignees_on_issue_id_and_user_id", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("issue_assignees", ["user_id"], {:name=>"index_issue_assignees_on_user_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("issue_metrics", {:force=>:cascade})
+   -> 0.0032s
+-- add_index("issue_metrics", ["issue_id"], {:name=>"index_issue_metrics", :using=>:btree})
+   -> 0.0029s
+-- create_table("issues", {:force=>:cascade})
+   -> 0.0051s
+-- add_index("issues", ["author_id"], {:name=>"index_issues_on_author_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("issues", ["confidential"], {:name=>"index_issues_on_confidential", :using=>:btree})
+   -> 0.0029s
+-- add_index("issues", ["description"], {:name=>"index_issues_on_description_trigram", :using=>:gin, :opclasses=>{"description"=>"gin_trgm_ops"}})
+   -> 0.0022s
+-- add_index("issues", ["milestone_id"], {:name=>"index_issues_on_milestone_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("issues", ["moved_to_id"], {:name=>"index_issues_on_moved_to_id", :where=>"(moved_to_id IS NOT NULL)", :using=>:btree})
+   -> 0.0030s
+-- add_index("issues", ["project_id", "created_at", "id", "state"], {:name=>"index_issues_on_project_id_and_created_at_and_id_and_state", :using=>:btree})
+   -> 0.0039s
+-- add_index("issues", ["project_id", "due_date", "id", "state"], {:name=>"idx_issues_on_project_id_and_due_date_and_id_and_state_partial", :where=>"(due_date IS NOT NULL)", :using=>:btree})
+   -> 0.0031s
+-- add_index("issues", ["project_id", "iid"], {:name=>"index_issues_on_project_id_and_iid", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("issues", ["project_id", "updated_at", "id", "state"], {:name=>"index_issues_on_project_id_and_updated_at_and_id_and_state", :using=>:btree})
+   -> 0.0035s
+-- add_index("issues", ["relative_position"], {:name=>"index_issues_on_relative_position", :using=>:btree})
+   -> 0.0030s
+-- add_index("issues", ["state"], {:name=>"index_issues_on_state", :using=>:btree})
+   -> 0.0027s
+-- add_index("issues", ["title"], {:name=>"index_issues_on_title_trigram", :using=>:gin, :opclasses=>{"title"=>"gin_trgm_ops"}})
+   -> 0.0021s
+-- add_index("issues", ["updated_at"], {:name=>"index_issues_on_updated_at", :using=>:btree})
+   -> 0.0030s
+-- add_index("issues", ["updated_by_id"], {:name=>"index_issues_on_updated_by_id", :where=>"(updated_by_id IS NOT NULL)", :using=>:btree})
+   -> 0.0028s
+-- create_table("keys", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("keys", ["fingerprint"], {:name=>"index_keys_on_fingerprint", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("keys", ["user_id"], {:name=>"index_keys_on_user_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("label_links", {:force=>:cascade})
+   -> 0.0041s
+-- add_index("label_links", ["label_id"], {:name=>"index_label_links_on_label_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("label_links", ["target_id", "target_type"], {:name=>"index_label_links_on_target_id_and_target_type", :using=>:btree})
+   -> 0.0028s
+-- create_table("label_priorities", {:force=>:cascade})
+   -> 0.0031s
+-- add_index("label_priorities", ["priority"], {:name=>"index_label_priorities_on_priority", :using=>:btree})
+   -> 0.0028s
+-- add_index("label_priorities", ["project_id", "label_id"], {:name=>"index_label_priorities_on_project_id_and_label_id", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- create_table("labels", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("labels", ["group_id", "project_id", "title"], {:name=>"index_labels_on_group_id_and_project_id_and_title", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("labels", ["project_id"], {:name=>"index_labels_on_project_id", :using=>:btree})
+   -> 0.0032s
+-- add_index("labels", ["template"], {:name=>"index_labels_on_template", :where=>"template", :using=>:btree})
+   -> 0.0027s
+-- add_index("labels", ["title"], {:name=>"index_labels_on_title", :using=>:btree})
+   -> 0.0030s
+-- add_index("labels", ["type", "project_id"], {:name=>"index_labels_on_type_and_project_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("lfs_objects", {:force=>:cascade})
+   -> 0.0040s
+-- add_index("lfs_objects", ["oid"], {:name=>"index_lfs_objects_on_oid", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- create_table("lfs_objects_projects", {:force=>:cascade})
+   -> 0.0035s
+-- add_index("lfs_objects_projects", ["project_id"], {:name=>"index_lfs_objects_projects_on_project_id", :using=>:btree})
+   -> 0.0025s
+-- create_table("lists", {:force=>:cascade})
+   -> 0.0033s
+-- add_index("lists", ["board_id", "label_id"], {:name=>"index_lists_on_board_id_and_label_id", :unique=>true, :using=>:btree})
+   -> 0.0026s
+-- add_index("lists", ["label_id"], {:name=>"index_lists_on_label_id", :using=>:btree})
+   -> 0.0026s
+-- create_table("members", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("members", ["access_level"], {:name=>"index_members_on_access_level", :using=>:btree})
+   -> 0.0028s
+-- add_index("members", ["invite_token"], {:name=>"index_members_on_invite_token", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- add_index("members", ["requested_at"], {:name=>"index_members_on_requested_at", :using=>:btree})
+   -> 0.0025s
+-- add_index("members", ["source_id", "source_type"], {:name=>"index_members_on_source_id_and_source_type", :using=>:btree})
+   -> 0.0027s
+-- add_index("members", ["user_id"], {:name=>"index_members_on_user_id", :using=>:btree})
+   -> 0.0026s
+-- create_table("merge_request_diff_commits", {:id=>false, :force=>:cascade})
+   -> 0.0027s
+-- add_index("merge_request_diff_commits", ["merge_request_diff_id", "relative_order"], {:name=>"index_merge_request_diff_commits_on_mr_diff_id_and_order", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("merge_request_diff_commits", ["sha"], {:name=>"index_merge_request_diff_commits_on_sha", :using=>:btree})
+   -> 0.0029s
+-- create_table("merge_request_diff_files", {:id=>false, :force=>:cascade})
+   -> 0.0027s
+-- add_index("merge_request_diff_files", ["merge_request_diff_id", "relative_order"], {:name=>"index_merge_request_diff_files_on_mr_diff_id_and_order", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- create_table("merge_request_diffs", {:force=>:cascade})
+   -> 0.0042s
+-- add_index("merge_request_diffs", ["merge_request_id", "id"], {:name=>"index_merge_request_diffs_on_merge_request_id_and_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("merge_request_metrics", {:force=>:cascade})
+   -> 0.0034s
+-- add_index("merge_request_metrics", ["first_deployed_to_production_at"], {:name=>"index_merge_request_metrics_on_first_deployed_to_production_at", :using=>:btree})
+   -> 0.0028s
+-- add_index("merge_request_metrics", ["merge_request_id"], {:name=>"index_merge_request_metrics", :using=>:btree})
+   -> 0.0025s
+-- add_index("merge_request_metrics", ["pipeline_id"], {:name=>"index_merge_request_metrics_on_pipeline_id", :using=>:btree})
+   -> 0.0026s
+-- create_table("merge_requests", {:force=>:cascade})
+   -> 0.0066s
+-- add_index("merge_requests", ["assignee_id"], {:name=>"index_merge_requests_on_assignee_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("merge_requests", ["author_id"], {:name=>"index_merge_requests_on_author_id", :using=>:btree})
+   -> 0.0026s
+-- add_index("merge_requests", ["created_at"], {:name=>"index_merge_requests_on_created_at", :using=>:btree})
+   -> 0.0026s
+-- add_index("merge_requests", ["description"], {:name=>"index_merge_requests_on_description_trigram", :using=>:gin, :opclasses=>{"description"=>"gin_trgm_ops"}})
+   -> 0.0020s
+-- add_index("merge_requests", ["head_pipeline_id"], {:name=>"index_merge_requests_on_head_pipeline_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("merge_requests", ["latest_merge_request_diff_id"], {:name=>"index_merge_requests_on_latest_merge_request_diff_id", :using=>:btree})
+   -> 0.0025s
+-- add_index("merge_requests", ["merge_user_id"], {:name=>"index_merge_requests_on_merge_user_id", :where=>"(merge_user_id IS NOT NULL)", :using=>:btree})
+   -> 0.0029s
+-- add_index("merge_requests", ["milestone_id"], {:name=>"index_merge_requests_on_milestone_id", :using=>:btree})
+   -> 0.0030s
+-- add_index("merge_requests", ["source_branch"], {:name=>"index_merge_requests_on_source_branch", :using=>:btree})
+   -> 0.0026s
+-- add_index("merge_requests", ["source_project_id", "source_branch"], {:name=>"index_merge_requests_on_source_project_and_branch_state_opened", :where=>"((state)::text = 'opened'::text)", :using=>:btree})
+   -> 0.0029s
+-- add_index("merge_requests", ["source_project_id", "source_branch"], {:name=>"index_merge_requests_on_source_project_id_and_source_branch", :using=>:btree})
+   -> 0.0031s
+-- add_index("merge_requests", ["target_branch"], {:name=>"index_merge_requests_on_target_branch", :using=>:btree})
+   -> 0.0028s
+-- add_index("merge_requests", ["target_project_id", "iid"], {:name=>"index_merge_requests_on_target_project_id_and_iid", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- add_index("merge_requests", ["target_project_id", "merge_commit_sha", "id"], {:name=>"index_merge_requests_on_tp_id_and_merge_commit_sha_and_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("merge_requests", ["title"], {:name=>"index_merge_requests_on_title", :using=>:btree})
+   -> 0.0026s
+-- add_index("merge_requests", ["title"], {:name=>"index_merge_requests_on_title_trigram", :using=>:gin, :opclasses=>{"title"=>"gin_trgm_ops"}})
+   -> 0.0020s
+-- add_index("merge_requests", ["updated_by_id"], {:name=>"index_merge_requests_on_updated_by_id", :where=>"(updated_by_id IS NOT NULL)", :using=>:btree})
+   -> 0.0029s
+-- create_table("merge_requests_closing_issues", {:force=>:cascade})
+   -> 0.0031s
+-- add_index("merge_requests_closing_issues", ["issue_id"], {:name=>"index_merge_requests_closing_issues_on_issue_id", :using=>:btree})
+   -> 0.0026s
+-- add_index("merge_requests_closing_issues", ["merge_request_id"], {:name=>"index_merge_requests_closing_issues_on_merge_request_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("milestones", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("milestones", ["description"], {:name=>"index_milestones_on_description_trigram", :using=>:gin, :opclasses=>{"description"=>"gin_trgm_ops"}})
+   -> 0.0022s
+-- add_index("milestones", ["due_date"], {:name=>"index_milestones_on_due_date", :using=>:btree})
+   -> 0.0033s
+-- add_index("milestones", ["group_id"], {:name=>"index_milestones_on_group_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("milestones", ["project_id", "iid"], {:name=>"index_milestones_on_project_id_and_iid", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("milestones", ["title"], {:name=>"index_milestones_on_title", :using=>:btree})
+   -> 0.0026s
+-- add_index("milestones", ["title"], {:name=>"index_milestones_on_title_trigram", :using=>:gin, :opclasses=>{"title"=>"gin_trgm_ops"}})
+   -> 0.0021s
+-- create_table("namespaces", {:force=>:cascade})
+   -> 0.0068s
+-- add_index("namespaces", ["created_at"], {:name=>"index_namespaces_on_created_at", :using=>:btree})
+   -> 0.0030s
+-- add_index("namespaces", ["name", "parent_id"], {:name=>"index_namespaces_on_name_and_parent_id", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("namespaces", ["name"], {:name=>"index_namespaces_on_name_trigram", :using=>:gin, :opclasses=>{"name"=>"gin_trgm_ops"}})
+   -> 0.0020s
+-- add_index("namespaces", ["owner_id"], {:name=>"index_namespaces_on_owner_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("namespaces", ["parent_id", "id"], {:name=>"index_namespaces_on_parent_id_and_id", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("namespaces", ["path"], {:name=>"index_namespaces_on_path", :using=>:btree})
+   -> 0.0031s
+-- add_index("namespaces", ["path"], {:name=>"index_namespaces_on_path_trigram", :using=>:gin, :opclasses=>{"path"=>"gin_trgm_ops"}})
+   -> 0.0019s
+-- add_index("namespaces", ["require_two_factor_authentication"], {:name=>"index_namespaces_on_require_two_factor_authentication", :using=>:btree})
+   -> 0.0029s
+-- add_index("namespaces", ["type"], {:name=>"index_namespaces_on_type", :using=>:btree})
+   -> 0.0032s
+-- create_table("notes", {:force=>:cascade})
+   -> 0.0055s
+-- add_index("notes", ["author_id"], {:name=>"index_notes_on_author_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("notes", ["commit_id"], {:name=>"index_notes_on_commit_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("notes", ["created_at"], {:name=>"index_notes_on_created_at", :using=>:btree})
+   -> 0.0029s
+-- add_index("notes", ["discussion_id"], {:name=>"index_notes_on_discussion_id", :using=>:btree})
+   -> 0.0029s
+-- add_index("notes", ["line_code"], {:name=>"index_notes_on_line_code", :using=>:btree})
+   -> 0.0029s
+-- add_index("notes", ["note"], {:name=>"index_notes_on_note_trigram", :using=>:gin, :opclasses=>{"note"=>"gin_trgm_ops"}})
+   -> 0.0024s
+-- add_index("notes", ["noteable_id", "noteable_type"], {:name=>"index_notes_on_noteable_id_and_noteable_type", :using=>:btree})
+   -> 0.0029s
+-- add_index("notes", ["noteable_type"], {:name=>"index_notes_on_noteable_type", :using=>:btree})
+   -> 0.0030s
+-- add_index("notes", ["project_id", "noteable_type"], {:name=>"index_notes_on_project_id_and_noteable_type", :using=>:btree})
+   -> 0.0027s
+-- add_index("notes", ["updated_at"], {:name=>"index_notes_on_updated_at", :using=>:btree})
+   -> 0.0026s
+-- create_table("notification_settings", {:force=>:cascade})
+   -> 0.0053s
+-- add_index("notification_settings", ["source_id", "source_type"], {:name=>"index_notification_settings_on_source_id_and_source_type", :using=>:btree})
+   -> 0.0028s
+-- add_index("notification_settings", ["user_id", "source_id", "source_type"], {:name=>"index_notifications_on_user_id_and_source_id_and_source_type", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("notification_settings", ["user_id"], {:name=>"index_notification_settings_on_user_id", :using=>:btree})
+   -> 0.0031s
+-- create_table("oauth_access_grants", {:force=>:cascade})
+   -> 0.0042s
+-- add_index("oauth_access_grants", ["token"], {:name=>"index_oauth_access_grants_on_token", :unique=>true, :using=>:btree})
+   -> 0.0031s
+-- create_table("oauth_access_tokens", {:force=>:cascade})
+   -> 0.0051s
+-- add_index("oauth_access_tokens", ["refresh_token"], {:name=>"index_oauth_access_tokens_on_refresh_token", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("oauth_access_tokens", ["resource_owner_id"], {:name=>"index_oauth_access_tokens_on_resource_owner_id", :using=>:btree})
+   -> 0.0025s
+-- add_index("oauth_access_tokens", ["token"], {:name=>"index_oauth_access_tokens_on_token", :unique=>true, :using=>:btree})
+   -> 0.0026s
+-- create_table("oauth_applications", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("oauth_applications", ["owner_id", "owner_type"], {:name=>"index_oauth_applications_on_owner_id_and_owner_type", :using=>:btree})
+   -> 0.0030s
+-- add_index("oauth_applications", ["uid"], {:name=>"index_oauth_applications_on_uid", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- create_table("oauth_openid_requests", {:force=>:cascade})
+   -> 0.0048s
+-- create_table("pages_domains", {:force=>:cascade})
+   -> 0.0052s
+-- add_index("pages_domains", ["domain"], {:name=>"index_pages_domains_on_domain", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- add_index("pages_domains", ["project_id"], {:name=>"index_pages_domains_on_project_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("personal_access_tokens", {:force=>:cascade})
+   -> 0.0056s
+-- add_index("personal_access_tokens", ["token"], {:name=>"index_personal_access_tokens_on_token", :unique=>true, :using=>:btree})
+   -> 0.0032s
+-- add_index("personal_access_tokens", ["user_id"], {:name=>"index_personal_access_tokens_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("project_authorizations", {:id=>false, :force=>:cascade})
+   -> 0.0018s
+-- add_index("project_authorizations", ["project_id"], {:name=>"index_project_authorizations_on_project_id", :using=>:btree})
+   -> 0.0033s
+-- add_index("project_authorizations", ["user_id", "project_id", "access_level"], {:name=>"index_project_authorizations_on_user_id_project_id_access_level", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("project_auto_devops", {:force=>:cascade})
+   -> 0.0043s
+-- add_index("project_auto_devops", ["project_id"], {:name=>"index_project_auto_devops_on_project_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("project_custom_attributes", {:force=>:cascade})
+   -> 0.0047s
+-- add_index("project_custom_attributes", ["key", "value"], {:name=>"index_project_custom_attributes_on_key_and_value", :using=>:btree})
+   -> 0.0030s
+-- add_index("project_custom_attributes", ["project_id", "key"], {:name=>"index_project_custom_attributes_on_project_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- create_table("project_features", {:force=>:cascade})
+   -> 0.0038s
+-- add_index("project_features", ["project_id"], {:name=>"index_project_features_on_project_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("project_group_links", {:force=>:cascade})
+   -> 0.0036s
+-- add_index("project_group_links", ["group_id"], {:name=>"index_project_group_links_on_group_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("project_group_links", ["project_id"], {:name=>"index_project_group_links_on_project_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("project_import_data", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("project_import_data", ["project_id"], {:name=>"index_project_import_data_on_project_id", :using=>:btree})
+   -> 0.0027s
+-- create_table("project_statistics", {:force=>:cascade})
+   -> 0.0046s
+-- add_index("project_statistics", ["namespace_id"], {:name=>"index_project_statistics_on_namespace_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("project_statistics", ["project_id"], {:name=>"index_project_statistics_on_project_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("projects", {:force=>:cascade})
+   -> 0.0090s
+-- add_index("projects", ["ci_id"], {:name=>"index_projects_on_ci_id", :using=>:btree})
+   -> 0.0033s
+-- add_index("projects", ["created_at"], {:name=>"index_projects_on_created_at", :using=>:btree})
+   -> 0.0030s
+-- add_index("projects", ["creator_id"], {:name=>"index_projects_on_creator_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("projects", ["description"], {:name=>"index_projects_on_description_trigram", :using=>:gin, :opclasses=>{"description"=>"gin_trgm_ops"}})
+   -> 0.0022s
+-- add_index("projects", ["last_activity_at"], {:name=>"index_projects_on_last_activity_at", :using=>:btree})
+   -> 0.0032s
+-- add_index("projects", ["last_repository_check_failed"], {:name=>"index_projects_on_last_repository_check_failed", :using=>:btree})
+   -> 0.0030s
+-- add_index("projects", ["last_repository_updated_at"], {:name=>"index_projects_on_last_repository_updated_at", :using=>:btree})
+   -> 0.0031s
+-- add_index("projects", ["name"], {:name=>"index_projects_on_name_trigram", :using=>:gin, :opclasses=>{"name"=>"gin_trgm_ops"}})
+   -> 0.0022s
+-- add_index("projects", ["namespace_id"], {:name=>"index_projects_on_namespace_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("projects", ["path"], {:name=>"index_projects_on_path", :using=>:btree})
+   -> 0.0028s
+-- add_index("projects", ["path"], {:name=>"index_projects_on_path_trigram", :using=>:gin, :opclasses=>{"path"=>"gin_trgm_ops"}})
+   -> 0.0023s
+-- add_index("projects", ["pending_delete"], {:name=>"index_projects_on_pending_delete", :using=>:btree})
+   -> 0.0029s
+-- add_index("projects", ["repository_storage"], {:name=>"index_projects_on_repository_storage", :using=>:btree})
+   -> 0.0026s
+-- add_index("projects", ["runners_token"], {:name=>"index_projects_on_runners_token", :using=>:btree})
+   -> 0.0034s
+-- add_index("projects", ["star_count"], {:name=>"index_projects_on_star_count", :using=>:btree})
+   -> 0.0028s
+-- add_index("projects", ["visibility_level"], {:name=>"index_projects_on_visibility_level", :using=>:btree})
+   -> 0.0027s
+-- create_table("protected_branch_merge_access_levels", {:force=>:cascade})
+   -> 0.0042s
+-- add_index("protected_branch_merge_access_levels", ["protected_branch_id"], {:name=>"index_protected_branch_merge_access", :using=>:btree})
+   -> 0.0029s
+-- create_table("protected_branch_push_access_levels", {:force=>:cascade})
+   -> 0.0037s
+-- add_index("protected_branch_push_access_levels", ["protected_branch_id"], {:name=>"index_protected_branch_push_access", :using=>:btree})
+   -> 0.0030s
+-- create_table("protected_branches", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("protected_branches", ["project_id"], {:name=>"index_protected_branches_on_project_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("protected_tag_create_access_levels", {:force=>:cascade})
+   -> 0.0037s
+-- add_index("protected_tag_create_access_levels", ["protected_tag_id"], {:name=>"index_protected_tag_create_access", :using=>:btree})
+   -> 0.0029s
+-- add_index("protected_tag_create_access_levels", ["user_id"], {:name=>"index_protected_tag_create_access_levels_on_user_id", :using=>:btree})
+   -> 0.0029s
+-- create_table("protected_tags", {:force=>:cascade})
+   -> 0.0051s
+-- add_index("protected_tags", ["project_id"], {:name=>"index_protected_tags_on_project_id", :using=>:btree})
+   -> 0.0034s
+-- create_table("push_event_payloads", {:id=>false, :force=>:cascade})
+   -> 0.0030s
+-- add_index("push_event_payloads", ["event_id"], {:name=>"index_push_event_payloads_on_event_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("redirect_routes", {:force=>:cascade})
+   -> 0.0049s
+-- add_index("redirect_routes", ["path"], {:name=>"index_redirect_routes_on_path", :unique=>true, :using=>:btree})
+   -> 0.0031s
+-- add_index("redirect_routes", ["source_type", "source_id"], {:name=>"index_redirect_routes_on_source_type_and_source_id", :using=>:btree})
+   -> 0.0034s
+-- create_table("releases", {:force=>:cascade})
+   -> 0.0043s
+-- add_index("releases", ["project_id", "tag"], {:name=>"index_releases_on_project_id_and_tag", :using=>:btree})
+   -> 0.0032s
+-- add_index("releases", ["project_id"], {:name=>"index_releases_on_project_id", :using=>:btree})
+   -> 0.0030s
+-- create_table("routes", {:force=>:cascade})
+   -> 0.0055s
+-- add_index("routes", ["path"], {:name=>"index_routes_on_path", :unique=>true, :using=>:btree})
+   -> 0.0028s
+-- add_index("routes", ["path"], {:name=>"index_routes_on_path_text_pattern_ops", :using=>:btree, :opclasses=>{"path"=>"varchar_pattern_ops"}})
+   -> 0.0026s
+-- add_index("routes", ["source_type", "source_id"], {:name=>"index_routes_on_source_type_and_source_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("sent_notifications", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("sent_notifications", ["reply_key"], {:name=>"index_sent_notifications_on_reply_key", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("services", {:force=>:cascade})
+   -> 0.0091s
+-- add_index("services", ["project_id"], {:name=>"index_services_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("services", ["template"], {:name=>"index_services_on_template", :using=>:btree})
+   -> 0.0031s
+-- create_table("snippets", {:force=>:cascade})
+   -> 0.0050s
+-- add_index("snippets", ["author_id"], {:name=>"index_snippets_on_author_id", :using=>:btree})
+   -> 0.0030s
+-- add_index("snippets", ["file_name"], {:name=>"index_snippets_on_file_name_trigram", :using=>:gin, :opclasses=>{"file_name"=>"gin_trgm_ops"}})
+   -> 0.0020s
+-- add_index("snippets", ["project_id"], {:name=>"index_snippets_on_project_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("snippets", ["title"], {:name=>"index_snippets_on_title_trigram", :using=>:gin, :opclasses=>{"title"=>"gin_trgm_ops"}})
+   -> 0.0020s
+-- add_index("snippets", ["updated_at"], {:name=>"index_snippets_on_updated_at", :using=>:btree})
+   -> 0.0026s
+-- add_index("snippets", ["visibility_level"], {:name=>"index_snippets_on_visibility_level", :using=>:btree})
+   -> 0.0026s
+-- create_table("spam_logs", {:force=>:cascade})
+   -> 0.0048s
+-- create_table("subscriptions", {:force=>:cascade})
+   -> 0.0041s
+-- add_index("subscriptions", ["subscribable_id", "subscribable_type", "user_id", "project_id"], {:name=>"index_subscriptions_on_subscribable_and_user_id_and_project_id", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- create_table("system_note_metadata", {:force=>:cascade})
+   -> 0.0040s
+-- add_index("system_note_metadata", ["note_id"], {:name=>"index_system_note_metadata_on_note_id", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- create_table("taggings", {:force=>:cascade})
+   -> 0.0047s
+-- add_index("taggings", ["tag_id", "taggable_id", "taggable_type", "context", "tagger_id", "tagger_type"], {:name=>"taggings_idx", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("taggings", ["taggable_id", "taggable_type", "context"], {:name=>"index_taggings_on_taggable_id_and_taggable_type_and_context", :using=>:btree})
+   -> 0.0025s
+-- create_table("tags", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("tags", ["name"], {:name=>"index_tags_on_name", :unique=>true, :using=>:btree})
+   -> 0.0026s
+-- create_table("timelogs", {:force=>:cascade})
+   -> 0.0033s
+-- add_index("timelogs", ["issue_id"], {:name=>"index_timelogs_on_issue_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("timelogs", ["merge_request_id"], {:name=>"index_timelogs_on_merge_request_id", :using=>:btree})
+   -> 0.0033s
+-- add_index("timelogs", ["user_id"], {:name=>"index_timelogs_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("todos", {:force=>:cascade})
+   -> 0.0043s
+-- add_index("todos", ["author_id"], {:name=>"index_todos_on_author_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("todos", ["commit_id"], {:name=>"index_todos_on_commit_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("todos", ["note_id"], {:name=>"index_todos_on_note_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("todos", ["project_id"], {:name=>"index_todos_on_project_id", :using=>:btree})
+   -> 0.0027s
+-- add_index("todos", ["target_type", "target_id"], {:name=>"index_todos_on_target_type_and_target_id", :using=>:btree})
+   -> 0.0028s
+-- add_index("todos", ["user_id"], {:name=>"index_todos_on_user_id", :using=>:btree})
+   -> 0.0026s
+-- create_table("trending_projects", {:force=>:cascade})
+   -> 0.0030s
+-- add_index("trending_projects", ["project_id"], {:name=>"index_trending_projects_on_project_id", :using=>:btree})
+   -> 0.0027s
+-- create_table("u2f_registrations", {:force=>:cascade})
+   -> 0.0048s
+-- add_index("u2f_registrations", ["key_handle"], {:name=>"index_u2f_registrations_on_key_handle", :using=>:btree})
+   -> 0.0029s
+-- add_index("u2f_registrations", ["user_id"], {:name=>"index_u2f_registrations_on_user_id", :using=>:btree})
+   -> 0.0028s
+-- create_table("uploads", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("uploads", ["checksum"], {:name=>"index_uploads_on_checksum", :using=>:btree})
+   -> 0.0028s
+-- add_index("uploads", ["model_id", "model_type"], {:name=>"index_uploads_on_model_id_and_model_type", :using=>:btree})
+   -> 0.0027s
+-- add_index("uploads", ["path"], {:name=>"index_uploads_on_path", :using=>:btree})
+   -> 0.0028s
+-- create_table("user_agent_details", {:force=>:cascade})
+   -> 0.0051s
+-- add_index("user_agent_details", ["subject_id", "subject_type"], {:name=>"index_user_agent_details_on_subject_id_and_subject_type", :using=>:btree})
+   -> 0.0028s
+-- create_table("user_custom_attributes", {:force=>:cascade})
+   -> 0.0044s
+-- add_index("user_custom_attributes", ["key", "value"], {:name=>"index_user_custom_attributes_on_key_and_value", :using=>:btree})
+   -> 0.0027s
+-- add_index("user_custom_attributes", ["user_id", "key"], {:name=>"index_user_custom_attributes_on_user_id_and_key", :unique=>true, :using=>:btree})
+   -> 0.0026s
+-- create_table("user_synced_attributes_metadata", {:force=>:cascade})
+   -> 0.0056s
+-- add_index("user_synced_attributes_metadata", ["user_id"], {:name=>"index_user_synced_attributes_metadata_on_user_id", :unique=>true, :using=>:btree})
+   -> 0.0027s
+-- create_table("users", {:force=>:cascade})
+   -> 0.0134s
+-- add_index("users", ["admin"], {:name=>"index_users_on_admin", :using=>:btree})
+   -> 0.0030s
+-- add_index("users", ["confirmation_token"], {:name=>"index_users_on_confirmation_token", :unique=>true, :using=>:btree})
+   -> 0.0029s
+-- add_index("users", ["created_at"], {:name=>"index_users_on_created_at", :using=>:btree})
+   -> 0.0034s
+-- add_index("users", ["email"], {:name=>"index_users_on_email", :unique=>true, :using=>:btree})
+   -> 0.0030s
+-- add_index("users", ["email"], {:name=>"index_users_on_email_trigram", :using=>:gin, :opclasses=>{"email"=>"gin_trgm_ops"}})
+   -> 0.0431s
+-- add_index("users", ["ghost"], {:name=>"index_users_on_ghost", :using=>:btree})
+   -> 0.0051s
+-- add_index("users", ["incoming_email_token"], {:name=>"index_users_on_incoming_email_token", :using=>:btree})
+   -> 0.0044s
+-- add_index("users", ["name"], {:name=>"index_users_on_name", :using=>:btree})
+   -> 0.0044s
+-- add_index("users", ["name"], {:name=>"index_users_on_name_trigram", :using=>:gin, :opclasses=>{"name"=>"gin_trgm_ops"}})
+   -> 0.0034s
+-- add_index("users", ["reset_password_token"], {:name=>"index_users_on_reset_password_token", :unique=>true, :using=>:btree})
+   -> 0.0044s
+-- add_index("users", ["rss_token"], {:name=>"index_users_on_rss_token", :using=>:btree})
+   -> 0.0046s
+-- add_index("users", ["state"], {:name=>"index_users_on_state", :using=>:btree})
+   -> 0.0040s
+-- add_index("users", ["username"], {:name=>"index_users_on_username", :using=>:btree})
+   -> 0.0046s
+-- add_index("users", ["username"], {:name=>"index_users_on_username_trigram", :using=>:gin, :opclasses=>{"username"=>"gin_trgm_ops"}})
+   -> 0.0044s
+-- create_table("users_star_projects", {:force=>:cascade})
+   -> 0.0055s
+-- add_index("users_star_projects", ["project_id"], {:name=>"index_users_star_projects_on_project_id", :using=>:btree})
+   -> 0.0037s
+-- add_index("users_star_projects", ["user_id", "project_id"], {:name=>"index_users_star_projects_on_user_id_and_project_id", :unique=>true, :using=>:btree})
+   -> 0.0044s
+-- create_table("web_hook_logs", {:force=>:cascade})
+   -> 0.0060s
+-- add_index("web_hook_logs", ["web_hook_id"], {:name=>"index_web_hook_logs_on_web_hook_id", :using=>:btree})
+   -> 0.0034s
+-- create_table("web_hooks", {:force=>:cascade})
+   -> 0.0120s
+-- add_index("web_hooks", ["project_id"], {:name=>"index_web_hooks_on_project_id", :using=>:btree})
+   -> 0.0038s
+-- add_index("web_hooks", ["type"], {:name=>"index_web_hooks_on_type", :using=>:btree})
+   -> 0.0036s
+-- add_foreign_key("boards", "projects", {:name=>"fk_f15266b5f9", :on_delete=>:cascade})
+   -> 0.0030s
+-- add_foreign_key("chat_teams", "namespaces", {:on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("ci_build_trace_section_names", "projects", {:on_delete=>:cascade})
+   -> 0.0022s
+-- add_foreign_key("ci_build_trace_sections", "ci_build_trace_section_names", {:column=>"section_name_id", :name=>"fk_264e112c66", :on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("ci_build_trace_sections", "ci_builds", {:column=>"build_id", :name=>"fk_4ebe41f502", :on_delete=>:cascade})
+   -> 0.0024s
+-- add_foreign_key("ci_build_trace_sections", "projects", {:on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("ci_builds", "ci_pipelines", {:column=>"auto_canceled_by_id", :name=>"fk_a2141b1522", :on_delete=>:nullify})
+   -> 0.0023s
+-- add_foreign_key("ci_builds", "ci_stages", {:column=>"stage_id", :name=>"fk_3a9eaa254d", :on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("ci_builds", "projects", {:name=>"fk_befce0568a", :on_delete=>:cascade})
+   -> 0.0024s
+-- add_foreign_key("ci_group_variables", "namespaces", {:column=>"group_id", :name=>"fk_33ae4d58d8", :on_delete=>:cascade})
+   -> 0.0024s
+-- add_foreign_key("ci_job_artifacts", "ci_builds", {:column=>"job_id", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("ci_job_artifacts", "projects", {:on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("ci_pipeline_schedule_variables", "ci_pipeline_schedules", {:column=>"pipeline_schedule_id", :name=>"fk_41c35fda51", :on_delete=>:cascade})
+   -> 0.0027s
+-- add_foreign_key("ci_pipeline_schedules", "projects", {:name=>"fk_8ead60fcc4", :on_delete=>:cascade})
+   -> 0.0022s
+-- add_foreign_key("ci_pipeline_schedules", "users", {:column=>"owner_id", :name=>"fk_9ea99f58d2", :on_delete=>:nullify})
+   -> 0.0025s
+-- add_foreign_key("ci_pipeline_variables", "ci_pipelines", {:column=>"pipeline_id", :name=>"fk_f29c5f4380", :on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("ci_pipelines", "ci_pipeline_schedules", {:column=>"pipeline_schedule_id", :name=>"fk_3d34ab2e06", :on_delete=>:nullify})
+   -> 0.0019s
+-- add_foreign_key("ci_pipelines", "ci_pipelines", {:column=>"auto_canceled_by_id", :name=>"fk_262d4c2d19", :on_delete=>:nullify})
+   -> 0.0029s
+-- add_foreign_key("ci_pipelines", "projects", {:name=>"fk_86635dbd80", :on_delete=>:cascade})
+   -> 0.0023s
+-- add_foreign_key("ci_runner_projects", "projects", {:name=>"fk_4478a6f1e4", :on_delete=>:cascade})
+   -> 0.0036s
+-- add_foreign_key("ci_stages", "ci_pipelines", {:column=>"pipeline_id", :name=>"fk_fb57e6cc56", :on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("ci_stages", "projects", {:name=>"fk_2360681d1d", :on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("ci_trigger_requests", "ci_triggers", {:column=>"trigger_id", :name=>"fk_b8ec8b7245", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("ci_triggers", "projects", {:name=>"fk_e3e63f966e", :on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("ci_triggers", "users", {:column=>"owner_id", :name=>"fk_e8e10d1964", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("ci_variables", "projects", {:name=>"fk_ada5eb64b3", :on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("cluster_platforms_kubernetes", "clusters", {:on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("cluster_projects", "clusters", {:on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("cluster_projects", "projects", {:on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("cluster_providers_gcp", "clusters", {:on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("clusters", "users", {:on_delete=>:nullify})
+   -> 0.0018s
+-- add_foreign_key("clusters_applications_helm", "clusters", {:on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("container_repositories", "projects")
+   -> 0.0020s
+-- add_foreign_key("deploy_keys_projects", "projects", {:name=>"fk_58a901ca7e", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("deployments", "projects", {:name=>"fk_b9a3851b82", :on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("environments", "projects", {:name=>"fk_d1c8c1da6a", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("events", "projects", {:on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("events", "users", {:column=>"author_id", :name=>"fk_edfd187b6f", :on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("fork_network_members", "fork_networks", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("fork_network_members", "projects", {:column=>"forked_from_project_id", :name=>"fk_b01280dae4", :on_delete=>:nullify})
+   -> 0.0019s
+-- add_foreign_key("fork_network_members", "projects", {:on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("fork_networks", "projects", {:column=>"root_project_id", :name=>"fk_e7b436b2b5", :on_delete=>:nullify})
+   -> 0.0018s
+-- add_foreign_key("forked_project_links", "projects", {:column=>"forked_to_project_id", :name=>"fk_434510edb0", :on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("gcp_clusters", "projects", {:on_delete=>:cascade})
+   -> 0.0029s
+-- add_foreign_key("gcp_clusters", "services", {:on_delete=>:nullify})
+   -> 0.0022s
+-- add_foreign_key("gcp_clusters", "users", {:on_delete=>:nullify})
+   -> 0.0019s
+-- add_foreign_key("gpg_key_subkeys", "gpg_keys", {:on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("gpg_keys", "users", {:on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("gpg_signatures", "gpg_key_subkeys", {:on_delete=>:nullify})
+   -> 0.0016s
+-- add_foreign_key("gpg_signatures", "gpg_keys", {:on_delete=>:nullify})
+   -> 0.0016s
+-- add_foreign_key("gpg_signatures", "projects", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("group_custom_attributes", "namespaces", {:column=>"group_id", :on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("issue_assignees", "issues", {:name=>"fk_b7d881734a", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("issue_assignees", "users", {:name=>"fk_5e0c8d9154", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("issue_metrics", "issues", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("issues", "issues", {:column=>"moved_to_id", :name=>"fk_a194299be1", :on_delete=>:nullify})
+   -> 0.0014s
+-- add_foreign_key("issues", "milestones", {:name=>"fk_96b1dd429c", :on_delete=>:nullify})
+   -> 0.0016s
+-- add_foreign_key("issues", "projects", {:name=>"fk_899c8f3231", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("issues", "users", {:column=>"author_id", :name=>"fk_05f1e72feb", :on_delete=>:nullify})
+   -> 0.0015s
+-- add_foreign_key("issues", "users", {:column=>"updated_by_id", :name=>"fk_ffed080f01", :on_delete=>:nullify})
+   -> 0.0017s
+-- add_foreign_key("label_priorities", "labels", {:on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("label_priorities", "projects", {:on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("labels", "namespaces", {:column=>"group_id", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("labels", "projects", {:name=>"fk_7de4989a69", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("lists", "boards", {:name=>"fk_0d3f677137", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("lists", "labels", {:name=>"fk_7a5553d60f", :on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("members", "users", {:name=>"fk_2e88fb7ce9", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("merge_request_diff_commits", "merge_request_diffs", {:on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("merge_request_diff_files", "merge_request_diffs", {:on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("merge_request_diffs", "merge_requests", {:name=>"fk_8483f3258f", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("merge_request_metrics", "ci_pipelines", {:column=>"pipeline_id", :on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("merge_request_metrics", "merge_requests", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("merge_request_metrics", "users", {:column=>"latest_closed_by_id", :name=>"fk_ae440388cc", :on_delete=>:nullify})
+   -> 0.0015s
+-- add_foreign_key("merge_request_metrics", "users", {:column=>"merged_by_id", :name=>"fk_7f28d925f3", :on_delete=>:nullify})
+   -> 0.0015s
+-- add_foreign_key("merge_requests", "ci_pipelines", {:column=>"head_pipeline_id", :name=>"fk_fd82eae0b9", :on_delete=>:nullify})
+   -> 0.0014s
+-- add_foreign_key("merge_requests", "merge_request_diffs", {:column=>"latest_merge_request_diff_id", :name=>"fk_06067f5644", :on_delete=>:nullify})
+   -> 0.0014s
+-- add_foreign_key("merge_requests", "milestones", {:name=>"fk_6a5165a692", :on_delete=>:nullify})
+   -> 0.0015s
+-- add_foreign_key("merge_requests", "projects", {:column=>"source_project_id", :name=>"fk_3308fe130c", :on_delete=>:nullify})
+   -> 0.0017s
+-- add_foreign_key("merge_requests", "projects", {:column=>"target_project_id", :name=>"fk_a6963e8447", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("merge_requests", "users", {:column=>"assignee_id", :name=>"fk_6149611a04", :on_delete=>:nullify})
+   -> 0.0016s
+-- add_foreign_key("merge_requests", "users", {:column=>"author_id", :name=>"fk_e719a85f8a", :on_delete=>:nullify})
+   -> 0.0017s
+-- add_foreign_key("merge_requests", "users", {:column=>"merge_user_id", :name=>"fk_ad525e1f87", :on_delete=>:nullify})
+   -> 0.0018s
+-- add_foreign_key("merge_requests", "users", {:column=>"updated_by_id", :name=>"fk_641731faff", :on_delete=>:nullify})
+   -> 0.0017s
+-- add_foreign_key("merge_requests_closing_issues", "issues", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("merge_requests_closing_issues", "merge_requests", {:on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("milestones", "namespaces", {:column=>"group_id", :name=>"fk_95650a40d4", :on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("milestones", "projects", {:name=>"fk_9bd0a0c791", :on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("notes", "projects", {:name=>"fk_99e097b079", :on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("oauth_openid_requests", "oauth_access_grants", {:column=>"access_grant_id", :name=>"fk_oauth_openid_requests_oauth_access_grants_access_grant_id"})
+   -> 0.0014s
+-- add_foreign_key("pages_domains", "projects", {:name=>"fk_ea2f6dfc6f", :on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("personal_access_tokens", "users")
+   -> 0.0016s
+-- add_foreign_key("project_authorizations", "projects", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("project_authorizations", "users", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("project_auto_devops", "projects", {:on_delete=>:cascade})
+   -> 0.0026s
+-- add_foreign_key("project_custom_attributes", "projects", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("project_features", "projects", {:name=>"fk_18513d9b92", :on_delete=>:cascade})
+   -> 0.0020s
+-- add_foreign_key("project_group_links", "projects", {:name=>"fk_daa8cee94c", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("project_import_data", "projects", {:name=>"fk_ffb9ee3a10", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("project_statistics", "projects", {:on_delete=>:cascade})
+   -> 0.0021s
+-- add_foreign_key("protected_branch_merge_access_levels", "protected_branches", {:name=>"fk_8a3072ccb3", :on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("protected_branch_push_access_levels", "protected_branches", {:name=>"fk_9ffc86a3d9", :on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("protected_branches", "projects", {:name=>"fk_7a9c6d93e7", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("protected_tag_create_access_levels", "namespaces", {:column=>"group_id"})
+   -> 0.0016s
+-- add_foreign_key("protected_tag_create_access_levels", "protected_tags", {:name=>"fk_f7dfda8c51", :on_delete=>:cascade})
+   -> 0.0013s
+-- add_foreign_key("protected_tag_create_access_levels", "users")
+   -> 0.0018s
+-- add_foreign_key("protected_tags", "projects", {:name=>"fk_8e4af87648", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("push_event_payloads", "events", {:name=>"fk_36c74129da", :on_delete=>:cascade})
+   -> 0.0013s
+-- add_foreign_key("releases", "projects", {:name=>"fk_47fe2a0596", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("services", "projects", {:name=>"fk_71cce407f9", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("snippets", "projects", {:name=>"fk_be41fd4bb7", :on_delete=>:cascade})
+   -> 0.0017s
+-- add_foreign_key("subscriptions", "projects", {:on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("system_note_metadata", "notes", {:name=>"fk_d83a918cb1", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("timelogs", "issues", {:name=>"fk_timelogs_issues_issue_id", :on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("timelogs", "merge_requests", {:name=>"fk_timelogs_merge_requests_merge_request_id", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("todos", "projects", {:name=>"fk_45054f9c45", :on_delete=>:cascade})
+   -> 0.0018s
+-- add_foreign_key("trending_projects", "projects", {:on_delete=>:cascade})
+   -> 0.0015s
+-- add_foreign_key("u2f_registrations", "users")
+   -> 0.0017s
+-- add_foreign_key("user_custom_attributes", "users", {:on_delete=>:cascade})
+   -> 0.0019s
+-- add_foreign_key("user_synced_attributes_metadata", "users", {:on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("users_star_projects", "projects", {:name=>"fk_22cd27ddfc", :on_delete=>:cascade})
+   -> 0.0016s
+-- add_foreign_key("web_hook_logs", "web_hooks", {:on_delete=>:cascade})
+   -> 0.0014s
+-- add_foreign_key("web_hooks", "projects", {:name=>"fk_0c8ca6d9d1", :on_delete=>:cascade})
+   -> 0.0017s
+-- initialize_schema_migrations_table()
+   -> 0.0112s
+$ JOB_NAME=( $CI_JOB_NAME )
+$ export CI_NODE_INDEX=${JOB_NAME[-2]}
+$ export CI_NODE_TOTAL=${JOB_NAME[-1]}
+$ export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+$ export KNAPSACK_GENERATE_REPORT=true
+$ export CACHE_CLASSES=true
+$ cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
+$ scripts/gitaly-test-spawn
+Gem.path: ["/root/.gem/ruby/2.3.0", "/usr/local/lib/ruby/gems/2.3.0", "/usr/local/bundle"]
+ENV['BUNDLE_GEMFILE']: nil
+ENV['RUBYOPT']: nil
+bundle config in /builds/gitlab-org/gitlab-ce
+scripts/gitaly-test-spawn:10:in `<main>': undefined local variable or method `gitaly_dir' for main:Object (NameError)
+Did you mean?  gitaly_dir
+Settings are listed in order of priority. The top value will be used.
+retry
+Set for your local app (/usr/local/bundle/config): 3
+
+path
+Set for your local app (/usr/local/bundle/config): "vendor"
+Set via BUNDLE_PATH: "/usr/local/bundle"
+
+jobs
+Set for your local app (/usr/local/bundle/config): "2"
+
+clean
+Set for your local app (/usr/local/bundle/config): "true"
+
+without
+Set for your local app (/usr/local/bundle/config): [:production]
+
+silence_root_warning
+Set via BUNDLE_SILENCE_ROOT_WARNING: true
+
+app_config
+Set via BUNDLE_APP_CONFIG: "/usr/local/bundle"
+
+install_flags
+Set via BUNDLE_INSTALL_FLAGS: "--without=production --jobs=2 --path=vendor --retry=3 --quiet"
+
+bin
+Set via BUNDLE_BIN: "/usr/local/bundle/bin"
+
+gemfile
+Set via BUNDLE_GEMFILE: "/builds/gitlab-org/gitlab-ce/Gemfile"
+
+section_end:1517486961:build_script
+section_start:1517486961:after_script
+section_end:1517486962:after_script
+section_start:1517486962:upload_artifacts
+Uploading artifacts...
+WARNING: coverage/: no matching files              
+knapsack/: found 5 matching files                  
+WARNING: tmp/capybara/: no matching files          
+Uploading artifacts to coordinator... ok            id=50551722 responseStatus=201 Created token=XkN753rp
+section_end:1517486963:upload_artifacts
+ERROR: Job failed: exit code 1
+
\ No newline at end of file

spec/javascripts/fixtures/jobs.rb

@@ -7,7 +7,7 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do
   let(:namespace) { create(:namespace, name: 'frontend-fixtures' )}
   let(:project) { create(:project_empty_repo, namespace: namespace, path: 'builds-project') }
   let(:pipeline) { create(:ci_empty_pipeline, project: project) }
-  let!(:build_with_artifacts) { create(:ci_build, :success, :artifacts, :trace, pipeline: pipeline, stage: 'test', artifacts_expire_at: Time.now + 18.months) }
+  let!(:build_with_artifacts) { create(:ci_build, :success, :artifacts, :trace_artifact, pipeline: pipeline, stage: 'test', artifacts_expire_at: Time.now + 18.months) }
   let!(:failed_build) { create(:ci_build, :failed, pipeline: pipeline, stage: 'build') }
   let!(:pending_build) { create(:ci_build, :pending, pipeline: pipeline, stage: 'deploy') }
 

spec/lib/gitlab/ci/trace_spec.rb

@@ -238,11 +238,98 @@ describe Gitlab::Ci::Trace do
     end
   end
 
+  describe '#read' do
+    shared_examples 'read successfully with IO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(IO)
+        end
+      end
+    end
+
+    shared_examples 'read successfully with StringIO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(StringIO)
+        end
+      end
+    end
+
+    shared_examples 'failed to read' do
+      it 'yields without source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_nil
+        end
+      end
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_id) exists' do
+      before do
+        expect(trace).to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_ci_id) exists' do
+      before do
+        expect(trace).to receive(:deprecated_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when db trace exists' do
+      before do
+        build.send(:write_attribute, :trace, "data")
+      end
+
+      it_behaves_like 'read successfully with StringIO'
+    end
+
+    context 'when no sources exist' do
+      it_behaves_like 'failed to read'
+    end
+  end
+
   describe 'trace handling' do
+    subject { trace.exist? }
+
     context 'trace does not exist' do
       it { expect(trace.exist?).to be(false) }
     end
 
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it { is_expected.to be_truthy }
+
+      context 'when the trace artifact has been erased' do
+        before do
+          trace.erase!
+        end
+
+        it { is_expected.to be_falsy }
+
+        it 'removes associations' do
+          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
+        end
+      end
+    end
+
     context 'new trace path is used' do
       before do
         trace.send(:ensure_directory)

spec/models/ci/build_spec.rb

@@ -726,7 +726,7 @@ describe Ci::Build do
 
     context 'build is erasable' do
       context 'new artifacts' do
-        let!(:build) { create(:ci_build, :trace, :success, :artifacts) }
+        let!(:build) { create(:ci_build, :trace_artifact, :success, :artifacts) }
 
         describe '#erase' do
           before do
@@ -760,7 +760,7 @@ describe Ci::Build do
         end
 
         describe '#erased?' do
-          let!(:build) { create(:ci_build, :trace, :success, :artifacts) }
+          let!(:build) { create(:ci_build, :trace_artifact, :success, :artifacts) }
           subject { build.erased? }
 
           context 'job has not been erased' do
@@ -795,7 +795,7 @@ describe Ci::Build do
     context 'old artifacts' do
       context 'build is erasable' do
         context 'new artifacts' do
-          let!(:build) { create(:ci_build, :trace, :success, :legacy_artifacts) }
+          let!(:build) { create(:ci_build, :trace_artifact, :success, :legacy_artifacts) }
 
           describe '#erase' do
             before do
@@ -829,7 +829,7 @@ describe Ci::Build do
           end
 
           describe '#erased?' do
-            let!(:build) { create(:ci_build, :trace, :success, :legacy_artifacts) }
+            let!(:build) { create(:ci_build, :trace_artifact, :success, :legacy_artifacts) }
             subject { build.erased? }
 
             context 'job has not been erased' do

spec/models/ci/job_artifact_spec.rb

@@ -12,6 +12,9 @@ describe Ci::JobArtifact do
   it { is_expected.to respond_to(:created_at) }
   it { is_expected.to respond_to(:updated_at) }
 
+  it { is_expected.to delegate_method(:open).to(:file) }
+  it { is_expected.to delegate_method(:exists?).to(:file) }
+
   describe 'callbacks' do
     subject { create(:ci_job_artifact, :archive) }
 

spec/requests/api/jobs_spec.rb

 require 'spec_helper'
 
 describe API::Jobs do
+  include HttpIOHelpers
+
   set(:project) do
     create(:project, :repository, public_builds: false)
   end
@@ -480,16 +482,43 @@ describe API::Jobs do
   end
 
   describe 'GET /projects/:id/jobs/:job_id/trace' do
-    let(:job) { create(:ci_build, :trace, pipeline: pipeline) }
-
     before do
       get api("/projects/#{project.id}/jobs/#{job.id}/trace", api_user)
     end
 
     context 'authorized user' do
-      it 'returns specific job trace' do
-        expect(response).to have_gitlab_http_status(200)
-        expect(response.body).to eq(job.trace.raw)
+      context 'when trace is in ObjectStorage' do
+        let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+        before do
+          stub_remote_trace_206
+          allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false }
+          allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url }
+          allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size }
+        end
+
+        it 'returns specific job trace' do
+          expect(response).to have_gitlab_http_status(200)
+          expect(response.body).to eq(job.trace.raw)
+        end
+      end
+
+      context 'when trace is artifact' do
+        let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) }
+
+        it 'returns specific job trace' do
+          expect(response).to have_gitlab_http_status(200)
+          expect(response.body).to eq(job.trace.raw)
+        end
+      end
+
+      context 'when trace is file' do
+        let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) }
+
+        it 'returns specific job trace' do
+          expect(response).to have_gitlab_http_status(200)
+          expect(response.body).to eq(job.trace.raw)
+        end
       end
     end
 
@@ -577,11 +606,11 @@ describe API::Jobs do
     end
 
     context 'job is erasable' do
-      let(:job) { create(:ci_build, :trace, :artifacts, :success, project: project, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :trace_artifact, :artifacts, :success, project: project, pipeline: pipeline) }
 
       it 'erases job content' do
         expect(response).to have_gitlab_http_status(201)
-        expect(job).not_to have_trace
+        expect(job.trace.exist?).to be_falsy
         expect(job.artifacts_file.exists?).to be_falsy
         expect(job.artifacts_metadata.exists?).to be_falsy
       end
@@ -595,7 +624,7 @@ describe API::Jobs do
     end
 
     context 'job is not erasable' do
-      let(:job) { create(:ci_build, :trace, project: project, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :trace_live, project: project, pipeline: pipeline) }
 
       it 'responds with forbidden' do
         expect(response).to have_gitlab_http_status(403)
@@ -604,7 +633,7 @@ describe API::Jobs do
 
     context 'when a developer erases a build' do
       let(:role) { :developer }
-      let(:job) { create(:ci_build, :trace, :artifacts, :success, project: project, pipeline: pipeline, user: owner) }
+      let(:job) { create(:ci_build, :trace_artifact, :artifacts, :success, project: project, pipeline: pipeline, user: owner) }
 
       context 'when the build was created by the developer' do
         let(:owner) { user }
@@ -627,7 +656,7 @@ describe API::Jobs do
 
     context 'artifacts did not expire' do
       let(:job) do
-        create(:ci_build, :trace, :artifacts, :success,
+        create(:ci_build, :trace_artifact, :artifacts, :success,
                project: project, pipeline: pipeline, artifacts_expire_at: Time.now + 7.days)
       end
 

spec/requests/api/runner_spec.rb

@@ -639,7 +639,7 @@ describe API::Runner do
     end
 
     describe 'PUT /api/v4/jobs/:id' do
-      let(:job) { create(:ci_build, :pending, :trace, pipeline: pipeline, runner_id: runner.id) }
+      let(:job) { create(:ci_build, :pending, :trace_live, pipeline: pipeline, runner_id: runner.id) }
 
       before do
         job.run!
@@ -681,11 +681,17 @@ describe API::Runner do
       end
 
       context 'when tace is given' do
-        it 'updates a running build' do
-          update_job(trace: 'BUILD TRACE UPDATED')
+        it 'creates a trace artifact' do
+          allow_any_instance_of(BuildFinishedWorker).to receive(:perform).with(job.id) do
+            CreateTraceArtifactWorker.new.perform(job.id)
+          end
+
+          update_job(state: 'success', trace: 'BUILD TRACE UPDATED')
 
+          job.reload
           expect(response).to have_gitlab_http_status(200)
-          expect(job.reload.trace.raw).to eq 'BUILD TRACE UPDATED'
+          expect(job.trace.raw).to eq 'BUILD TRACE UPDATED'
+          expect(job.job_artifacts_trace.open.read).to eq 'BUILD TRACE UPDATED'
         end
       end
 
@@ -714,7 +720,7 @@ describe API::Runner do
     end
 
     describe 'PATCH /api/v4/jobs/:id/trace' do
-      let(:job) { create(:ci_build, :running, :trace, runner_id: runner.id, pipeline: pipeline) }
+      let(:job) { create(:ci_build, :running, :trace_live, runner_id: runner.id, pipeline: pipeline) }
       let(:headers) { { API::Helpers::Runner::JOB_TOKEN_HEADER => job.token, 'Content-Type' => 'text/plain' } }
       let(:headers_with_range) { headers.merge({ 'Content-Range' => '11-20' }) }
       let(:update_interval) { 10.seconds.to_i }
@@ -775,7 +781,7 @@ describe API::Runner do
 
         context 'when project for the build has been deleted' do
           let(:job) do
-            create(:ci_build, :running, :trace, runner_id: runner.id, pipeline: pipeline) do |job|
+            create(:ci_build, :running, :trace_live, runner_id: runner.id, pipeline: pipeline) do |job|
               job.project.update(pending_delete: true)
             end
           end

spec/requests/api/v3/builds_spec.rb

@@ -356,7 +356,7 @@ describe API::V3::Builds do
   end
 
   describe 'GET /projects/:id/builds/:build_id/trace' do
-    let(:build) { create(:ci_build, :trace, pipeline: pipeline) }
+    let(:build) { create(:ci_build, :trace_live, pipeline: pipeline) }
 
     before do
       get v3_api("/projects/#{project.id}/builds/#{build.id}/trace", api_user)
@@ -451,7 +451,7 @@ describe API::V3::Builds do
     end
 
     context 'job is erasable' do
-      let(:build) { create(:ci_build, :trace, :artifacts, :success, project: project, pipeline: pipeline) }
+      let(:build) { create(:ci_build, :trace_artifact, :artifacts, :success, project: project, pipeline: pipeline) }
 
       it 'erases job content' do
         expect(response.status).to eq 201
@@ -467,7 +467,7 @@ describe API::V3::Builds do
     end
 
     context 'job is not erasable' do
-      let(:build) { create(:ci_build, :trace, project: project, pipeline: pipeline) }
+      let(:build) { create(:ci_build, :trace_live, project: project, pipeline: pipeline) }
 
       it 'responds with forbidden' do
         expect(response.status).to eq 403
@@ -482,7 +482,7 @@ describe API::V3::Builds do
 
     context 'artifacts did not expire' do
       let(:build) do
-        create(:ci_build, :trace, :artifacts, :success,
+        create(:ci_build, :trace_artifact, :artifacts, :success,
                project: project, pipeline: pipeline, artifacts_expire_at: Time.now + 7.days)
       end
 

spec/services/ci/create_trace_artifact_service_spec.rb

+require 'spec_helper'
+
+describe Ci::CreateTraceArtifactService do
+  describe '#execute' do
+    subject { described_class.new(nil, nil).execute(job) }
+
+    let(:job) { create(:ci_build) }
+
+    context 'when the job does not have trace artifact' do
+      context 'when the job has a trace file' do
+        before do
+          allow_any_instance_of(Gitlab::Ci::Trace)
+            .to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
+
+          allow_any_instance_of(JobArtifactUploader).to receive(:move_to_cache) { false }
+          allow_any_instance_of(JobArtifactUploader).to receive(:move_to_store) { false }
+        end
+
+        it 'creates trace artifact' do
+          expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+          expect(job.job_artifacts_trace.read_attribute(:file)).to eq('sample_trace')
+        end
+
+        context 'when the job has already had trace artifact' do
+          before do
+            create(:ci_job_artifact, :trace, job: job)
+          end
+
+          it 'does not create trace artifact' do
+            expect { subject }.not_to change { Ci::JobArtifact.count }
+          end
+        end
+      end
+
+      context 'when the job does not have a trace file' do
+        it 'does not create trace artifact' do
+          expect { subject }.not_to change { Ci::JobArtifact.count }
+        end
+      end
+    end
+  end
+end

spec/services/ci/retry_build_service_spec.rb

@@ -17,7 +17,8 @@ describe Ci::RetryBuildService do
     %i[id status user token coverage trace runner artifacts_expire_at
        artifacts_file artifacts_metadata artifacts_size created_at
        updated_at started_at finished_at queued_at erased_by
-       erased_at auto_canceled_by job_artifacts job_artifacts_archive job_artifacts_metadata].freeze
+       erased_at auto_canceled_by job_artifacts job_artifacts_archive
+       job_artifacts_metadata job_artifacts_trace].freeze
 
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags trace_sections
@@ -37,7 +38,7 @@ describe Ci::RetryBuildService do
     let(:build) do
       create(:ci_build, :failed, :artifacts, :expired, :erased,
              :queued, :coverage, :tags, :allowed_to_fail, :on_tag,
-             :triggered, :trace, :teardown_environment,
+             :triggered, :trace_artifact, :teardown_environment,
              description: 'my-job', stage: 'test',  pipeline: pipeline,
              auto_canceled_by: create(:ci_empty_pipeline, project: project)) do |build|
                ##

spec/support/http_io/http_io_helpers.rb

+module HttpIOHelpers
+  def stub_remote_trace_206
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 206) }
+  end
+
+  def stub_remote_trace_200
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return { |request| remote_trace_response(request, 200) }
+  end
+
+  def stub_remote_trace_500
+    WebMock.stub_request(:get, remote_trace_url)
+      .to_return(status: [500, "Internal Server Error"])
+  end
+
+  def remote_trace_url
+    "http://trace.com/trace"
+  end
+
+  def remote_trace_response(request, responce_status)
+    range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/)
+
+    {
+      status: responce_status,
+      headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i),
+      body: range_trace_body(range[1].to_i, range[2].to_i)
+    }
+  end
+
+  def remote_trace_response_headers(responce_status, from, to)
+    headers = { 'Content-Type' => 'text/plain' }
+
+    if responce_status == 206
+      headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}")
+    end
+
+    headers
+  end
+
+  def range_trace_body(from, to)
+    remote_trace_body[from..to]
+  end
+
+  def remote_trace_body
+    @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace'))
+  end
+
+  def remote_trace_size
+    remote_trace_body.length
+  end
+
+  def set_smaller_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks / 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+
+  def set_larger_buffer_size_than(file_size)
+    blocks = (file_size / 128)
+    new_size = (blocks * 2) * 128
+    stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size)
+  end
+end

spec/uploaders/job_artifact_uploader_spec.rb

@@ -23,6 +23,33 @@ describe JobArtifactUploader do
                     store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z]
   end
 
+  describe '#open' do
+    subject { uploader.open }
+
+    context 'when trace is stored in File storage' do
+      context 'when file exists' do
+        let(:file) do
+          fixture_file_upload(
+            Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
+        end
+
+        before do
+          uploader.store!(file)
+        end
+
+        it 'returns io stream' do
+          is_expected.to be_a(IO)
+        end
+      end
+
+      context 'when file does not exist' do
+        it 'returns nil' do
+          is_expected.to be_nil
+        end
+      end
+    end
+  end
+
   context 'file is stored in valid local_path' do
     let(:file) do
       fixture_file_upload(

spec/workers/build_finished_worker_spec.rb

@@ -6,17 +6,15 @@ describe BuildFinishedWorker do
       let!(:build) { create(:ci_build) }
 
       it 'calculates coverage and calls hooks' do
-        expect(BuildCoverageWorker)
+        expect(BuildTraceSectionsWorker)
           .to receive(:new).ordered.and_call_original
-        expect(BuildHooksWorker)
+        expect(BuildCoverageWorker)
           .to receive(:new).ordered.and_call_original
 
-        expect(BuildTraceSectionsWorker)
-          .to receive(:perform_async)
-        expect_any_instance_of(BuildCoverageWorker)
-          .to receive(:perform)
-        expect_any_instance_of(BuildHooksWorker)
-          .to receive(:perform)
+        expect_any_instance_of(BuildTraceSectionsWorker).to receive(:perform)
+        expect_any_instance_of(BuildCoverageWorker).to receive(:perform)
+        expect(BuildHooksWorker).to receive(:perform_async)
+        expect(CreateTraceArtifactWorker).to receive(:perform_async)
 
         described_class.new.perform(build.id)
       end

spec/workers/create_trace_artifact_worker_spec.rb

+require 'spec_helper'
+
+describe CreateTraceArtifactWorker do
+  describe '#perform' do
+    subject { described_class.new.perform(job&.id) }
+
+    context 'when job is found' do
+      let(:job) { create(:ci_build) }
+
+      it 'executes service' do
+        expect_any_instance_of(Ci::CreateTraceArtifactService)
+          .to receive(:execute).with(job)
+
+        subject
+      end
+    end
+
+    context 'when job is not found' do
+      let(:job) { nil }
+
+      it 'does not execute service' do
+        expect_any_instance_of(Ci::CreateTraceArtifactService)
+          .not_to receive(:execute)
+
+        subject
+      end
+    end
+  end
+end