Merge branch 'live-trace-v2' into 'master'

New CI Job live-trace architecture (v2)

Closes #44935

See merge request gitlab-org/gitlab-ce!18169

.flayignore

@@ -9,3 +9,4 @@ lib/gitlab/gitaly_client/operation_service.rb
 lib/gitlab/background_migration/*
 app/models/project_services/kubernetes_service.rb
 lib/gitlab/workhorse.rb
+lib/gitlab/ci/trace/chunked_io.rb

app/models/ci/build.rb

@@ -19,6 +19,7 @@ module Ci
 
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
+    has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id

app/models/ci/build_trace_chunk.rb

+module Ci
+  class BuildTraceChunk < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
+
+    after_destroy :redis_delete_data, if: :redis?
+
+    default_value_for :data_store, :redis
+
+    WriteError = Class.new(StandardError)
+
+    CHUNK_SIZE = 128.kilobytes
+    CHUNK_REDIS_TTL = 1.week
+    WRITE_LOCK_RETRY = 10
+    WRITE_LOCK_SLEEP = 0.01.seconds
+    WRITE_LOCK_TTL = 1.minute
+
+    enum data_store: {
+      redis: 1,
+      db: 2
+    }
+
+    ##
+    # Data is memoized for optimizing #size and #end_offset
+    def data
+      @data ||= get_data.to_s
+    end
+
+    def truncate(offset = 0)
+      raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
+      return if offset == size # Skip the following process as it doesn't affect anything
+
+      self.append("", offset)
+    end
+
+    def append(new_data, offset)
+      raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
+      raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize)
+
+      set_data(data.byteslice(0, offset) + new_data)
+    end
+
+    def size
+      data&.bytesize.to_i
+    end
+
+    def start_offset
+      chunk_index * CHUNK_SIZE
+    end
+
+    def end_offset
+      start_offset + size
+    end
+
+    def range
+      (start_offset...end_offset)
+    end
+
+    def use_database!
+      in_lock do
+        break if db?
+        break unless size > 0
+
+        self.update!(raw_data: data, data_store: :db)
+        redis_delete_data
+      end
+    end
+
+    private
+
+    def get_data
+      if redis?
+        redis_data
+      elsif db?
+        raw_data
+      else
+        raise 'Unsupported data store'
+      end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default
+    end
+
+    def set_data(value)
+      raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE
+
+      in_lock do
+        if redis?
+          redis_set_data(value)
+        elsif db?
+          self.raw_data = value
+        else
+          raise 'Unsupported data store'
+        end
+
+        @data = value
+
+        save! if changed?
+      end
+
+      schedule_to_db if full?
+    end
+
+    def schedule_to_db
+      return if db?
+
+      Ci::BuildTraceChunkFlushWorker.perform_async(id)
+    end
+
+    def full?
+      size == CHUNK_SIZE
+    end
+
+    def redis_data
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.get(redis_data_key)
+      end
+    end
+
+    def redis_set_data(data)
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL)
+      end
+    end
+
+    def redis_delete_data
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.del(redis_data_key)
+      end
+    end
+
+    def redis_data_key
+      "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}"
+    end
+
+    def redis_lock_key
+      "trace_write:#{build_id}:chunks:#{chunk_index}"
+    end
+
+    def in_lock
+      lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: WRITE_LOCK_TTL)
+      retry_count = 0
+
+      until uuid = lease.try_obtain
+        # Keep trying until we obtain the lease. To prevent hammering Redis too
+        # much we'll wait for a bit between retries.
+        sleep(WRITE_LOCK_SLEEP)
+        break if WRITE_LOCK_RETRY < (retry_count += 1)
+      end
+
+      raise WriteError, 'Failed to obtain write lock' unless uuid
+
+      self.reload if self.persisted?
+      return yield
+    ensure
+      Gitlab::ExclusiveLease.cancel(redis_lock_key, uuid)
+    end
+  end
+end

app/workers/all_queues.yml

@@ -52,6 +52,7 @@
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
 - pipeline_background:archive_trace
+- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
 - pipeline_default:pipeline_metrics

app/workers/ci/build_trace_chunk_flush_worker.rb

+module Ci
+  class BuildTraceChunkFlushWorker
+    include ApplicationWorker
+    include PipelineBackgroundQueue
+
+    def perform(build_trace_chunk_id)
+      ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|
+        build_trace_chunk.use_database!
+      end
+    end
+  end
+end

changelogs/unreleased/live-trace-v2.yml

+---
+title: New CI Job live-trace architecture
+merge_request: 18169
+author:
+type: changed

db/migrate/20180326202229_create_ci_build_trace_chunks.rb

+class CreateCiBuildTraceChunks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_build_trace_chunks, id: :bigserial do |t|
+      t.integer :build_id, null: false
+      t.integer :chunk_index, null: false
+      t.integer :data_store, null: false
+      t.binary :raw_data
+
+      t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade
+      t.index [:build_id, :chunk_index], unique: true
+    end
+  end
+end

db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
+
+class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    LimitsCiBuildTraceChunksRawDataForMysql.new.up
+  end
+end

db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb

+class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
+  def up
+    return unless Gitlab::Database.mysql?
+
+    # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB)
+    # Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB
+    change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
+  end
+end

db/schema.rb

@@ -253,6 +253,15 @@ ActiveRecord::Schema.define(version: 20180503200320) do
 
   add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
 
+  create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t|
+    t.integer "build_id", null: false
+    t.integer "chunk_index", null: false
+    t.integer "data_store", null: false
+    t.binary "raw_data"
+  end
+
+  add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree
+
   create_table "ci_build_trace_section_names", force: :cascade do |t|
     t.integer "project_id", null: false
     t.string "name", null: false
@@ -2110,6 +2119,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
   add_foreign_key "boards", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
+  add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade
   add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade

doc/administration/job_traces.md

@@ -40,3 +40,98 @@ To change the location where the job logs will be stored, follow the steps below
 
 [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
 [restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab"
+
+## New live trace architecture
+
+> [Introduced][ce-18169] in GitLab 10.4.
+
+> **Notes**:
+- This feature is still Beta, which could impact GitLab.com/on-premises instances, and in the worst case scenario, traces will be lost.
+- This feature is still being discussed in [an issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/46097) for the performance improvements.
+- This feature is off by default. Please check below how to enable/disable this featrue.
+
+**What is "live trace"?**
+
+Job trace that is sent by runner while jobs are running. You can see live trace in job pages UI.
+The live traces are archived once job finishes.
+
+**What is new architecture?**
+
+So far, when GitLab Runner sends a job trace to GitLab-Rails, traces have been saved to file storage as text files.
+This was a problem for [Cloud Native-compatible GitLab application](https://gitlab.com/gitlab-com/migration/issues/23) where GitLab had to rely on File Storage.
+
+This new live trace architecture stores chunks of traces in Redis and database instead of file storage.
+Redis is used as first-class storage, and it stores up-to 128kB. Once the full chunk is sent it will be flushed to database. Afterwhile, the data in Redis and database will be archived to ObjectStorage.
+
+Here is the detailed data flow.
+
+1. GitLab Runner picks a job from GitLab-Rails
+1. GitLab Runner sends a piece of trace to GitLab-Rails
+1. GitLab-Rails appends the data to Redis
+1. If the data in Redis is fulfilled 128kB, the data is flushed to Database.
+1. 2.~4. is continued until the job is finished
+1. Once the job is finished, GitLab-Rails schedules a sidekiq worker to archive the trace
+1. The sidekiq worker archives the trace to Object Storage, and cleanup the trace in Redis and Database
+
+**How to check if it's on or off?**
+
+```ruby
+Feature.enabled?('ci_enable_live_trace')
+```
+
+**How to enable?**
+
+```ruby
+Feature.enable('ci_enable_live_trace')
+```
+
+>**Note:**
+The transition period will be handled gracefully. Upcoming traces will be generated with the new architecture, and on-going live traces will stay with the legacy architecture (i.e. on-going live traces won't be re-generated forcibly with the new architecture).
+
+**How to disable?**
+
+```ruby
+Feature.disable('ci_enable_live_trace')
+```
+
+>**Note:**
+The transition period will be handled gracefully. Upcoming traces will be generated with the legacy architecture, and on-going live traces will stay with the new architecture (i.e. on-going live traces won't be re-generated forcibly with the legacy architecture).
+
+**Redis namespace:**
+
+`Gitlab::Redis::SharedState`
+
+**Potential impact:**
+
+- This feature could incur data loss:
+  - Case 1: When all data in Redis are accidentally flushed.
+    - On-going live traces could be recovered by re-sending traces (This is supported by all versions of GitLab Runner)
+    - Finished jobs which has not archived live traces will lose the last part (~128kB) of trace data.
+  - Case 2: When sidekiq workers failed to archive (e.g. There was a bug that prevents archiving process, Sidekiq inconsistancy, etc):
+    - Currently all trace data in Redis will be deleted after one week. If the sidekiq workers can't finish by the expiry date, the part of trace data will be lost.
+- This feature could consume all memory on Redis instance. If the number of jobs is 1000, 128MB (128kB * 1000) is consumed.
+- This feature could pressure Database replication lag. `INSERT` are generated to indicate that we have trace chunk. `UPDATE` with 128kB of data is issued once we receive multiple chunks.
+- and so on
+
+**How to test?**
+
+We're currently evaluating this feature on dev.gitalb.org or staging.gitlab.com to verify this features. Here is the list of tests/measurements.
+
+- Features:
+  - Live traces should be visible on job pages
+  - Archived traces should be visible on job pages
+  - Live traces should be archived to Object storage
+  - Live traces should be cleaned up after archived
+  - etc
+- Performance:
+  - Schedule 1000~10000 jobs and let GitLab-runners process concurrently. Measure memoery presssure, IO load, etc.
+  - etc
+- Failover:
+  - Simulate Redis outage
+  - etc
+
+**How to verify the correctnesss?**
+
+- TBD
+
+[ce-44935]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18169

lib/api/runner.rb

@@ -153,9 +153,20 @@ module API
         content_range = request.headers['Content-Range']
         content_range = content_range.split('-')
 
-        stream_size = job.trace.append(request.body.read, content_range[0].to_i)
-        if stream_size < 0
-          break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
+        # TODO:
+        # it seems that `Content-Range` as formatted by runner is wrong,
+        # the `byte_end` should point to final byte, but it points byte+1
+        # that means that we have to calculate end of body,
+        # as we cannot use `content_length[1]`
+        # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275
+
+        body_data = request.body.read
+        body_start = content_range[0].to_i
+        body_end = body_start + body_data.bytesize
+
+        stream_size = job.trace.append(body_data, body_start)
+        unless stream_size == body_end
+          break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{stream_size}" })
         end
 
         status 202

lib/gitlab/ci/trace.rb

@@ -36,16 +36,16 @@ module Gitlab
       end
 
       def set(data)
-        write do |stream|
+        write('w+b') do |stream|
           data = job.hide_secrets(data)
           stream.set(data)
         end
       end
 
       def append(data, offset)
-        write do |stream|
+        write('a+b') do |stream|
           current_length = stream.size
-          break -current_length unless current_length == offset
+          break current_length unless current_length == offset
 
           data = job.hide_secrets(data)
           stream.append(data, offset)
@@ -54,13 +54,15 @@ module Gitlab
       end
 
       def exist?
-        trace_artifact&.exists? || current_path.present? || old_trace.present?
+        trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
           if trace_artifact
             trace_artifact.open
+          elsif job.trace_chunks.any?
+            Gitlab::Ci::Trace::ChunkedIO.new(job)
           elsif current_path
             File.open(current_path, "rb")
           elsif old_trace
@@ -73,9 +75,15 @@ module Gitlab
         stream&.close
       end
 
-      def write
+      def write(mode)
         stream = Gitlab::Ci::Trace::Stream.new do
-          File.open(ensure_path, "a+b")
+          if current_path
+            File.open(current_path, mode)
+          elsif Feature.enabled?('ci_enable_live_trace')
+            Gitlab::Ci::Trace::ChunkedIO.new(job)
+          else
+            File.open(ensure_path, mode)
+          end
         end
 
         yield(stream).tap do
@@ -92,6 +100,7 @@ module Gitlab
           FileUtils.rm(trace_path, force: true)
         end
 
+        job.trace_chunks.destroy_all
         job.erase_old_trace!
       end
 
@@ -99,7 +108,12 @@ module Gitlab
         raise ArchiveError, 'Already archived' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        if current_path
+        if job.trace_chunks.any?
+          Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
+            archive_stream!(stream)
+            stream.destroy!
+          end
+        elsif current_path
           File.open(current_path) do |stream|
             archive_stream!(stream)
             FileUtils.rm(current_path)
@@ -116,7 +130,7 @@ module Gitlab
 
       def archive_stream!(stream)
         clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
-          create_job_trace!(job, clone_path)
+          create_build_trace!(job, clone_path)
         end
       end
 
@@ -132,7 +146,7 @@ module Gitlab
         end
       end
 
-      def create_job_trace!(job, path)
+      def create_build_trace!(job, path)
         File.open(path) do |stream|
           job.create_job_artifacts_trace!(
             project: job.project,

lib/gitlab/ci/trace/chunked_io.rb

+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  module Ci
+    class Trace
+      class ChunkedIO
+        CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE
+
+        FailedToGetChunkError = Class.new(StandardError)
+
+        attr_reader :build
+        attr_reader :tell, :size
+        attr_reader :chunk, :chunk_range
+
+        alias_method :pos, :tell
+
+        def initialize(build, &block)
+          @build = build
+          @chunks_cache = []
+          @tell = 0
+          @size = calculate_size
+          yield self if block_given?
+        end
+
+        def close
+          # no-op
+        end
+
+        def binmode
+          # no-op
+        end
+
+        def binmode?
+          true
+        end
+
+        def seek(pos, where = IO::SEEK_SET)
+          new_pos =
+            case where
+            when IO::SEEK_END
+              size + pos
+            when IO::SEEK_SET
+              pos
+            when IO::SEEK_CUR
+              tell + pos
+            else
+              -1
+            end
+
+          raise ArgumentError, 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+          @tell = new_pos
+        end
+
+        def eof?
+          tell == size
+        end
+
+        def each_line
+          until eof?
+            line = readline
+            break if line.nil?
+
+            yield(line)
+          end
+        end
+
+        def read(length = nil, outbuf = "")
+          out = ""
+
+          length ||= size - tell
+
+          until length <= 0 || eof?
+            data = chunk_slice_from_offset
+            break if data.empty?
+
+            chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min
+            chunk_data = data.byteslice(0, chunk_bytes)
+
+            out << chunk_data
+            @tell += chunk_data.bytesize
+            length -= chunk_data.bytesize
+          end
+
+          # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality
+          if outbuf
+            outbuf.slice!(0, outbuf.bytesize)
+            outbuf << out
+          end
+
+          out
+        end
+
+        def readline
+          out = ""
+
+          until eof?
+            data = chunk_slice_from_offset
+            new_line = data.index("\n")
+
+            if !new_line.nil?
+              out << data[0..new_line]
+              @tell += new_line + 1
+              break
+            else
+              out << data
+              @tell += data.bytesize
+            end
+          end
+
+          out
+        end
+
+        def write(data)
+          start_pos = tell
+
+          while tell < start_pos + data.bytesize
+            # get slice from current offset till the end where it falls into chunk
+            chunk_bytes = CHUNK_SIZE - chunk_offset
+            chunk_data = data.byteslice(tell - start_pos, chunk_bytes)
+
+            # append data to chunk, overwriting from that point
+            ensure_chunk.append(chunk_data, chunk_offset)
+
+            # move offsets within buffer
+            @tell += chunk_data.bytesize
+            @size = [size, tell].max
+          end
+
+          tell - start_pos
+        ensure
+          invalidate_chunk_cache
+        end
+
+        def truncate(offset)
+          raise ArgumentError, 'Outside of file' if offset > size || offset < 0
+          return if offset == size # Skip the following process as it doesn't affect anything
+
+          @tell = offset
+          @size = offset
+
+          # remove all next chunks
+          trace_chunks.where('chunk_index > ?', chunk_index).destroy_all
+
+          # truncate current chunk
+          current_chunk.truncate(chunk_offset)
+        ensure
+          invalidate_chunk_cache
+        end
+
+        def flush
+          # no-op
+        end
+
+        def present?
+          true
+        end
+
+        def destroy!
+          trace_chunks.destroy_all
+          @tell = @size = 0
+        ensure
+          invalidate_chunk_cache
+        end
+
+        private
+
+        ##
+        # The below methods are not implemented in IO class
+        #
+        def in_range?
+          @chunk_range&.include?(tell)
+        end
+
+        def chunk_slice_from_offset
+          unless in_range?
+            current_chunk.tap do |chunk|
+              raise FailedToGetChunkError unless chunk
+
+              @chunk = chunk.data
+              @chunk_range = chunk.range
+            end
+          end
+
+          @chunk[chunk_offset..CHUNK_SIZE]
+        end
+
+        def chunk_offset
+          tell % CHUNK_SIZE
+        end
+
+        def chunk_index
+          tell / CHUNK_SIZE
+        end
+
+        def chunk_start
+          chunk_index * CHUNK_SIZE
+        end
+
+        def chunk_end
+          [chunk_start + CHUNK_SIZE, size].min
+        end
+
+        def invalidate_chunk_cache
+          @chunks_cache = []
+        end
+
+        def current_chunk
+          @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
+        end
+
+        def build_chunk
+          @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
+        end
+
+        def ensure_chunk
+          current_chunk || build_chunk
+        end
+
+        def trace_chunks
+          ::Ci::BuildTraceChunk.where(build: build)
+        end
+
+        def calculate_size
+          trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/trace/stream.rb

@@ -39,6 +39,8 @@ module Gitlab
         end
 
         def append(data, offset)
+          data = data.force_encoding(Encoding::BINARY)
+
           stream.truncate(offset)
           stream.seek(0, IO::SEEK_END)
           stream.write(data)
@@ -46,8 +48,11 @@ module Gitlab
         end
 
         def set(data)
-          truncate(0)
+          data = data.force_encoding(Encoding::BINARY)
+
+          stream.seek(0, IO::SEEK_SET)
           stream.write(data)
+          stream.truncate(data.bytesize)
           stream.flush()
         end
 
@@ -127,11 +132,11 @@ module Gitlab
             buf += debris
             debris, *lines = buf.each_line.to_a
             lines.reverse_each do |line|
-              yield(line.force_encoding('UTF-8'))
+              yield(line.force_encoding(Encoding.default_external))
             end
           end
 
-          yield(debris.force_encoding('UTF-8')) unless debris.empty?
+          yield(debris.force_encoding(Encoding.default_external)) unless debris.empty?
         end
 
         def read_backward(length)

lib/tasks/migrate/add_limits_mysql.rake

 require Rails.root.join('db/migrate/limits_to_mysql')
 require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -8,4 +9,5 @@ task add_limits_mysql: :environment do
   LimitsToMysql.new.up
   MarkdownCacheLimitsToMysql.new.up
   MergeRequestDiffFileLimitsToMysql.new.up
+  LimitsCiBuildTraceChunksRawDataForMysql.new.up
 end

spec/controllers/projects/jobs_controller_spec.rb

 # coding: utf-8
 require 'spec_helper'
 
-describe Projects::JobsController do
+describe Projects::JobsController, :clean_gitlab_redis_shared_state do
   include ApiHelpers
   include HttpIOHelpers
 
@@ -10,6 +10,7 @@ describe Projects::JobsController do
   let(:user) { create(:user) }
 
   before do
+    stub_feature_flags(ci_enable_live_trace: true)
     stub_not_protect_default_branch
   end
 

spec/factories/ci/build_trace_chunks.rb

+FactoryBot.define do
+  factory :ci_build_trace_chunk, class: Ci::BuildTraceChunk do
+    build factory: :ci_build
+    chunk_index 0
+    data_store :redis
+  end
+end

spec/features/projects/jobs_spec.rb

 require 'spec_helper'
 require 'tempfile'
 
-feature 'Jobs' do
+feature 'Jobs', :clean_gitlab_redis_shared_state do
   let(:user) { create(:user) }
   let(:user_access_level) { :developer }
   let(:project) { create(:project, :repository) }
@@ -282,7 +282,7 @@ feature 'Jobs' do
         it 'loads job trace' do
           expect(page).to have_content 'BUILD TRACE'
 
-          job.trace.write do |stream|
+          job.trace.write('a+b') do |stream|
             stream.append(' and more trace', 11)
           end
 
@@ -593,44 +593,6 @@ feature 'Jobs' do
       end
     end
 
-    context 'storage form' do
-      let(:existing_file) { Tempfile.new('existing-trace-file').path }
-
-      before do
-        job.run!
-      end
-
-      context 'when job has trace in file', :js do
-        before do
-          allow_any_instance_of(Gitlab::Ci::Trace)
-            .to receive(:paths)
-            .and_return([existing_file])
-        end
-
-        it 'sends the right headers' do
-          requests = inspect_requests(inject_headers: { 'X-Sendfile-Type' => 'X-Sendfile' }) do
-            visit raw_project_job_path(project, job)
-          end
-          expect(requests.first.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
-          expect(requests.first.response_headers['X-Sendfile']).to eq(existing_file)
-        end
-      end
-
-      context 'when job has trace in the database', :js do
-        before do
-          allow_any_instance_of(Gitlab::Ci::Trace)
-            .to receive(:paths)
-            .and_return([])
-
-          visit project_job_path(project, job)
-        end
-
-        it 'sends the right headers' do
-          expect(page).not_to have_selector('.js-raw-link-controller')
-        end
-      end
-    end
-
     context "when visiting old URL" do
       let(:raw_job_url) do
         raw_project_job_path(project, job)

spec/lib/gitlab/ci/trace/chunked_io_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
+  include ChunkedIOHelpers
+
+  set(:build) { create(:ci_build, :running) }
+  let(:chunked_io) { described_class.new(build) }
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
+  context "#initialize" do
+    context 'when a chunk exists' do
+      before do
+        build.trace.set('ABC')
+      end
+
+      it { expect(chunked_io.size).to eq(3) }
+    end
+
+    context 'when two chunks exist' do
+      before do
+        stub_buffer_size(4)
+        build.trace.set('ABCDEF')
+      end
+
+      it { expect(chunked_io.size).to eq(6) }
+    end
+
+    context 'when no chunks exists' do
+      it { expect(chunked_io.size).to eq(0) }
+    end
+  end
+
+  context "#seek" do
+    subject { chunked_io.seek(pos, where) }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    context 'when moves pos to end of the file' do
+      let(:pos) { 0 }
+      let(:where) { IO::SEEK_END }
+
+      it { is_expected.to eq(sample_trace_raw.bytesize) }
+    end
+
+    context 'when moves pos to middle of the file' do
+      let(:pos) { sample_trace_raw.bytesize / 2 }
+      let(:where) { IO::SEEK_SET }
+
+      it { is_expected.to eq(pos) }
+    end
+
+    context 'when moves pos around' do
+      it 'matches the result' do
+        expect(chunked_io.seek(0)).to eq(0)
+        expect(chunked_io.seek(100, IO::SEEK_CUR)).to eq(100)
+        expect { chunked_io.seek(sample_trace_raw.bytesize + 1, IO::SEEK_CUR) }
+          .to raise_error('new position is outside of file')
+      end
+    end
+  end
+
+  context "#eof?" do
+    subject { chunked_io.eof? }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    context 'when current pos is at end of the file' do
+      before do
+        chunked_io.seek(sample_trace_raw.bytesize, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when current pos is not at end of the file' do
+      before do
+        chunked_io.seek(0, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  context "#each_line" do
+    let(:string_io) { StringIO.new(sample_trace_raw) }
+
+    context 'when buffer size is smaller than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it 'yields lines' do
+        expect { |b| chunked_io.each_line(&b) }
+          .to yield_successive_args(*string_io.each_line.to_a)
+      end
+    end
+
+    context 'when buffer size is larger than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize * 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it 'calls get_chunk only once' do
+        expect_any_instance_of(Gitlab::Ci::Trace::ChunkedIO)
+          .to receive(:current_chunk).once.and_call_original
+
+        chunked_io.each_line { |line| }
+      end
+    end
+  end
+
+  context "#read" do
+    subject { chunked_io.read(length) }
+
+    context 'when read the whole size' do
+      let(:length) { nil }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it { is_expected.to eq(sample_trace_raw) }
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it { is_expected.to eq(sample_trace_raw) }
+      end
+    end
+
+    context 'when read only first 100 bytes' do
+      let(:length) { 100 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw.byteslice(0, length))
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw.byteslice(0, length))
+        end
+      end
+    end
+
+    context 'when tries to read oversize' do
+      let(:length) { sample_trace_raw.bytesize + 1000 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw)
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw)
+        end
+      end
+    end
+
+    context 'when tries to read 0 bytes' do
+      let(:length) { 0 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to be_empty
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to be_empty
+        end
+      end
+    end
+  end
+
+  context "#readline" do
+    subject { chunked_io.readline }
+
+    let(:string_io) { StringIO.new(sample_trace_raw) }
+
+    shared_examples 'all line matching' do
+      it do
+        (0...sample_trace_raw.lines.count).each do
+          expect(chunked_io.readline).to eq(string_io.readline)
+        end
+      end
+    end
+
+    context 'when buffer size is smaller than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when buffer size is larger than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize * 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when pos is at middle of the file' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+
+        chunked_io.seek(chunked_io.size / 2)
+        string_io.seek(string_io.size / 2)
+      end
+
+      it 'reads from pos' do
+        expect(chunked_io.readline).to eq(string_io.readline)
+      end
+    end
+  end
+
+  context "#write" do
+    subject { chunked_io.write(data) }
+
+    let(:data) { sample_trace_raw }
+
+    context 'when data does not exist' do
+      shared_examples 'writes a trace' do
+        it do
+          is_expected.to eq(data.bytesize)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(data)
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(data.bytesize / 2)
+        end
+
+        it_behaves_like 'writes a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(data.bytesize * 2)
+        end
+
+        it_behaves_like 'writes a trace'
+      end
+    end
+
+    context 'when data already exists' do
+      let(:exist_data) { 'exist data' }
+
+      shared_examples 'appends a trace' do
+        it do
+          chunked_io.seek(0, IO::SEEK_END)
+          is_expected.to eq(data.bytesize)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(exist_data + data)
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(exist_data)
+        end
+
+        it_behaves_like 'appends a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(exist_data)
+        end
+
+        it_behaves_like 'appends a trace'
+      end
+    end
+  end
+
+  context "#truncate" do
+    let(:offset) { 10 }
+
+    context 'when data does not exist' do
+      shared_examples 'truncates a trace' do
+        it do
+          chunked_io.truncate(offset)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(sample_trace_raw.byteslice(0, offset))
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it_behaves_like 'truncates a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it_behaves_like 'truncates a trace'
+      end
+    end
+  end
+
+  context "#destroy!" do
+    subject { chunked_io.destroy! }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    it 'deletes' do
+      expect { subject }.to change { chunked_io.size }
+        .from(sample_trace_raw.bytesize).to(0)
+
+      expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0)
+    end
+  end
+end

spec/lib/gitlab/ci/trace/stream_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Ci::Trace::Stream do
+describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
+  set(:build) { create(:ci_build, :running) }
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
   describe 'delegates' do
     subject { described_class.new { nil } }
 
@@ -11,16 +17,10 @@ describe Gitlab::Ci::Trace::Stream do
     it { is_expected.to delegate_method(:path).to(:stream) }
     it { is_expected.to delegate_method(:truncate).to(:stream) }
     it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) }
-    it { is_expected.to delegate_method(:file?).to(:path).as(:present?) }
   end
 
   describe '#limit' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new((1..8).to_a.join("\n"))
-      end
-    end
-
+    shared_examples_for 'limits' do
       it 'if size is larger we start from beginning' do
         stream.limit(20)
 
@@ -70,21 +70,32 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#append' do
-    let(:tempfile) { Tempfile.new }
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new((1..8).to_a.join("\n"))
+        end
+      end
 
+      it_behaves_like 'limits'
+    end
+
+    context 'when stream is ChunkedIO' do
       let(:stream) do
         described_class.new do
-        tempfile.write("12345678")
-        tempfile.rewind
-        tempfile
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write((1..8).to_a.join("\n"))
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
         end
       end
 
-    after do
-      tempfile.unlink
+      it_behaves_like 'limits'
+    end
   end
 
+  describe '#append' do
+    shared_examples_for 'appends' do
       it "truncates and append content" do
         stream.append("89", 4)
         stream.seek(0)
@@ -105,13 +116,40 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#set' do
+    context 'when stream is Tempfile' do
+      let(:tempfile) { Tempfile.new }
+
       let(:stream) do
         described_class.new do
-        StringIO.new("12345678")
+          tempfile.write("12345678")
+          tempfile.rewind
+          tempfile
         end
       end
 
+      after do
+        tempfile.unlink
+      end
+
+      it_behaves_like 'appends'
+    end
+
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write('12345678')
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'appends'
+    end
+  end
+
+  describe '#set' do
+    shared_examples_for 'sets' do
       before do
         stream.set("8901")
       end
@@ -124,15 +162,32 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#raw' do
-    let(:path) { __FILE__ }
-    let(:lines) { File.readlines(path) }
+    context 'when stream is StringIO' do
       let(:stream) do
         described_class.new do
-        File.open(path)
+          StringIO.new("12345678")
+        end
+      end
+
+      it_behaves_like 'sets'
+    end
+
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write('12345678')
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
         end
       end
 
+      it_behaves_like 'sets'
+    end
+  end
+
+  describe '#raw' do
+    shared_examples_for 'sets' do
       it 'returns all contents if last_lines is not specified' do
         result = stream.raw
 
@@ -163,13 +218,35 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#html_with_state' do
+    let(:path) { __FILE__ }
+    let(:lines) { File.readlines(path) }
+
+    context 'when stream is File' do
       let(:stream) do
         described_class.new do
-        StringIO.new("1234")
+          File.open(path)
+        end
+      end
+
+      it_behaves_like 'sets'
+    end
+
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write(File.binread(path))
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'sets'
     end
   end
 
+  describe '#html_with_state' do
+    shared_examples_for 'html_with_states' do
       it 'returns html content with state' do
         result = stream.html_with_state
 
@@ -193,13 +270,32 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#html' do
+    context 'when stream is StringIO' do
       let(:stream) do
         described_class.new do
-        StringIO.new("12\n34\n56")
+          StringIO.new("1234")
         end
       end
 
+      it_behaves_like 'html_with_states'
+    end
+
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write("1234")
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'html_with_states'
+    end
+  end
+
+  describe '#html' do
+    shared_examples_for 'htmls' do
       it "returns html" do
         expect(stream.html).to eq("12<br>34<br>56")
       end
@@ -209,15 +305,32 @@ describe Gitlab::Ci::Trace::Stream do
       end
     end
 
-  describe '#extract_coverage' do
+    context 'when stream is StringIO' do
       let(:stream) do
         described_class.new do
-        StringIO.new(data)
+          StringIO.new("12\n34\n56")
         end
       end
 
-    subject { stream.extract_coverage(regex) }
+      it_behaves_like 'htmls'
+    end
 
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write("12\n34\n56")
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'htmls'
+    end
+  end
+
+  describe '#extract_coverage' do
+    shared_examples_for 'extract_coverages' do
       context 'valid content & regex' do
         let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' }
         let(:regex) { '\(\d+.\d+\%\) covered' }
@@ -344,4 +457,30 @@ describe Gitlab::Ci::Trace::Stream do
         end
       end
     end
+
+    subject { stream.extract_coverage(regex) }
+
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new(data)
+        end
+      end
+
+      it_behaves_like 'extract_coverages'
+    end
+
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write(data)
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'extract_coverages'
+    end
+  end
 end

spec/lib/gitlab/ci/trace_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Ci::Trace do
+describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
   let(:build) { create(:ci_build) }
   let(:trace) { described_class.new(build) }
 
@@ -9,552 +9,19 @@ describe Gitlab::Ci::Trace do
     it { expect(trace).to delegate_method(:old_trace).to(:job) }
   end
 
-  describe '#html' do
+  context 'when live trace feature is disabled' do
     before do
-      trace.set("12\n34")
+      stub_feature_flags(ci_enable_live_trace: false)
     end
 
-    it "returns formatted html" do
-      expect(trace.html).to eq("12<br>34")
+    it_behaves_like 'trace with disabled live trace feature'
   end
 
-    it "returns last line of formatted html" do
-      expect(trace.html(last_lines: 1)).to eq("34")
-    end
-  end
-
-  describe '#raw' do
-    before do
-      trace.set("12\n34")
-    end
-
-    it "returns raw output" do
-      expect(trace.raw).to eq("12\n34")
-    end
-
-    it "returns last line of raw output" do
-      expect(trace.raw(last_lines: 1)).to eq("34")
-    end
-  end
-
-  describe '#extract_coverage' do
-    let(:regex) { '\(\d+.\d+\%\) covered' }
-
-    context 'matching coverage' do
-      before do
-        trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
-      end
-
-      it "returns valid coverage" do
-        expect(trace.extract_coverage(regex)).to eq("98.29")
-      end
-    end
-
-    context 'no coverage' do
-      before do
-        trace.set('No coverage')
-      end
-
-      it 'returs nil' do
-        expect(trace.extract_coverage(regex)).to be_nil
-      end
-    end
-  end
-
-  describe '#extract_sections' do
-    let(:log) { 'No sections' }
-    let(:sections) { trace.extract_sections }
-
-    before do
-      trace.set(log)
-    end
-
-    context 'no sections' do
-      it 'returs []' do
-        expect(trace.extract_sections).to eq([])
-      end
-    end
-
-    context 'multiple sections available' do
-      let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) }
-      let(:sections_data) do
-        [
-          { name: 'prepare_script', lines: 2, duration: 3.seconds },
-          { name: 'get_sources', lines: 4, duration: 1.second },
-          { name: 'restore_cache', lines: 0, duration: 0.seconds },
-          { name: 'download_artifacts', lines: 0, duration: 0.seconds },
-          { name: 'build_script', lines: 2, duration: 1.second },
-          { name: 'after_script', lines: 0, duration: 0.seconds },
-          { name: 'archive_cache', lines: 0, duration: 0.seconds },
-          { name: 'upload_artifacts', lines: 0, duration: 0.seconds }
-        ]
-      end
-
-      it "returns valid sections" do
-        expect(sections).not_to be_empty
-        expect(sections.size).to eq(sections_data.size),
-                                 "expected #{sections_data.size} sections, got #{sections.size}"
-
-        buff = StringIO.new(log)
-        sections.each_with_index do |s, i|
-          expected = sections_data[i]
-
-          expect(s[:name]).to eq(expected[:name])
-          expect(s[:date_end] - s[:date_start]).to eq(expected[:duration])
-
-          buff.seek(s[:byte_start], IO::SEEK_SET)
-          length = s[:byte_end] - s[:byte_start]
-          lines = buff.read(length).count("\n")
-          expect(lines).to eq(expected[:lines])
-        end
-      end
-    end
-
-    context 'logs contains "section_start"' do
-      let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"}
-
-      it "returns only one section" do
-        expect(sections).not_to be_empty
-        expect(sections.size).to eq(1)
-
-        section = sections[0]
-        expect(section[:name]).to eq('a_section')
-        expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section"
-      end
-    end
-
-    context 'missing section_end' do
-      let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-
-    context 'missing section_start' do
-      let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-
-    context 'inverted section_start section_end' do
-      let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-  end
-
-  describe '#set' do
-    before do
-      trace.set("12")
-    end
-
-    it "returns trace" do
-      expect(trace.raw).to eq("12")
-    end
-
-    context 'overwrite trace' do
-      before do
-        trace.set("34")
-      end
-
-      it "returns new trace" do
-        expect(trace.raw).to eq("34")
-      end
-    end
-
-    context 'runners token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.project.update(runners_token: token)
-        trace.set(token)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-
-    context 'hides build token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.update(token: token)
-        trace.set(token)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-  end
-
-  describe '#append' do
-    before do
-      trace.set("1234")
-    end
-
-    it "returns correct trace" do
-      expect(trace.append("56", 4)).to eq(6)
-      expect(trace.raw).to eq("123456")
-    end
-
-    context 'tries to append trace at different offset' do
-      it "fails with append" do
-        expect(trace.append("56", 2)).to eq(-4)
-        expect(trace.raw).to eq("1234")
-      end
-    end
-
-    context 'runners token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.project.update(runners_token: token)
-        trace.append(token, 0)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-
-    context 'build token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.update(token: token)
-        trace.append(token, 0)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-  end
-
-  describe '#read' do
-    shared_examples 'read successfully with IO' do
-      it 'yields with source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_a(IO)
-        end
-      end
-    end
-
-    shared_examples 'read successfully with StringIO' do
-      it 'yields with source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_a(StringIO)
-        end
-      end
-    end
-
-    shared_examples 'failed to read' do
-      it 'yields without source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_nil
-        end
-      end
-    end
-
-    context 'when trace artifact exists' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when current_path (with project_id) exists' do
-      before do
-        expect(trace).to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
-      end
-
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when current_path (with project_ci_id) exists' do
+  context 'when live trace feature is enabled' do
     before do
-        expect(trace).to receive(:deprecated_path) { expand_fixture_path('trace/sample_trace') }
+      stub_feature_flags(ci_enable_live_trace: true)
     end
 
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when db trace exists' do
-      before do
-        build.send(:write_attribute, :trace, "data")
-      end
-
-      it_behaves_like 'read successfully with StringIO'
-    end
-
-    context 'when no sources exist' do
-      it_behaves_like 'failed to read'
-    end
-  end
-
-  describe 'trace handling' do
-    subject { trace.exist? }
-
-    context 'trace does not exist' do
-      it { expect(trace.exist?).to be(false) }
-    end
-
-    context 'when trace artifact exists' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it { is_expected.to be_truthy }
-
-      context 'when the trace artifact has been erased' do
-        before do
-          trace.erase!
-        end
-
-        it { is_expected.to be_falsy }
-
-        it 'removes associations' do
-          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
-        end
-      end
-    end
-
-    context 'new trace path is used' do
-      before do
-        trace.send(:ensure_directory)
-
-        File.open(trace.send(:default_path), "w") do |file|
-          file.write("data")
-        end
-      end
-
-      it "trace exist" do
-        expect(trace.exist?).to be(true)
-      end
-
-      it "can be erased" do
-        trace.erase!
-        expect(trace.exist?).to be(false)
-      end
-    end
-
-    context 'deprecated path' do
-      let(:path) { trace.send(:deprecated_path) }
-
-      context 'with valid ci_id' do
-        before do
-          build.project.update(ci_id: 1000)
-
-          FileUtils.mkdir_p(File.dirname(path))
-
-          File.open(path, "w") do |file|
-            file.write("data")
-          end
-        end
-
-        it "trace exist" do
-          expect(trace.exist?).to be(true)
-        end
-
-        it "can be erased" do
-          trace.erase!
-          expect(trace.exist?).to be(false)
-        end
-      end
-
-      context 'without valid ci_id' do
-        it "does not return deprecated path" do
-          expect(path).to be_nil
-        end
-      end
-    end
-
-    context 'stored in database' do
-      before do
-        build.send(:write_attribute, :trace, "data")
-      end
-
-      it "trace exist" do
-        expect(trace.exist?).to be(true)
-      end
-
-      it "can be erased" do
-        trace.erase!
-        expect(trace.exist?).to be(false)
-      end
-
-      it "returns database data" do
-        expect(trace.raw).to eq("data")
-      end
-    end
-  end
-
-  describe '#archive!' do
-    subject { trace.archive! }
-
-    shared_examples 'archive trace file' do
-      it do
-        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
-        expect(File.exist?(src_path)).to be_falsy
-        expect(src_checksum)
-          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
-        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
-      end
-    end
-
-    shared_examples 'source trace file stays intact' do |error:|
-      it do
-        expect { subject }.to raise_error(error)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace).to be_nil
-        expect(File.exist?(src_path)).to be_truthy
-      end
-    end
-
-    shared_examples 'archive trace in database' do
-      it do
-        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
-        expect(build.old_trace).to be_nil
-        expect(src_checksum)
-          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
-        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
-      end
-    end
-
-    shared_examples 'source trace in database stays intact' do |error:|
-      it do
-        expect { subject }.to raise_error(error)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace).to be_nil
-        expect(build.old_trace).to eq(trace_content)
-      end
-    end
-
-    context 'when job does not have trace artifact' do
-      context 'when trace file stored in default path' do
-        let!(:build) { create(:ci_build, :success, :trace_live) }
-        let!(:src_path) { trace.read { |s| s.path } }
-        let!(:src_checksum) { Digest::SHA256.file(src_path).hexdigest }
-
-        it_behaves_like 'archive trace file'
-
-        context 'when failed to create clone file' do
-          before do
-            allow(IO).to receive(:copy_stream).and_return(0)
-          end
-
-          it_behaves_like 'source trace file stays intact', error: Gitlab::Ci::Trace::ArchiveError
-        end
-
-        context 'when failed to create job artifact record' do
-          before do
-            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          it_behaves_like 'source trace file stays intact', error: ActiveRecord::RecordInvalid
-        end
-      end
-
-      context 'when trace is stored in database' do
-        let(:build) { create(:ci_build, :success) }
-        let(:trace_content) { 'Sample trace' }
-        let!(:src_checksum) { Digest::SHA256.hexdigest(trace_content) }
-
-        before do
-          build.update_column(:trace, trace_content)
-        end
-
-        it_behaves_like 'archive trace in database'
-
-        context 'when failed to create clone file' do
-          before do
-            allow(IO).to receive(:copy_stream).and_return(0)
-          end
-
-          it_behaves_like 'source trace in database stays intact', error: Gitlab::Ci::Trace::ArchiveError
-        end
-
-        context 'when failed to create job artifact record' do
-          before do
-            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          it_behaves_like 'source trace in database stays intact', error: ActiveRecord::RecordInvalid
-        end
-
-        context 'when there is a validation error on Ci::Build' do
-          before do
-            allow_any_instance_of(Ci::Build).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::Build).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          context "when erase old trace with 'save'" do
-            before do
-              build.send(:write_attribute, :trace, nil)
-              build.save
-            end
-
-            it 'old trace is not deleted' do
-              build.reload
-              expect(build.trace.raw).to eq(trace_content)
-            end
-          end
-
-          it_behaves_like 'archive trace in database'
-        end
-      end
-    end
-
-    context 'when job has trace artifact' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it 'does not archive' do
-        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
-        expect { subject }.to raise_error('Already archived')
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-      end
-    end
-
-    context 'when job is not finished yet' do
-      let!(:build) { create(:ci_build, :running, :trace_live) }
-
-      it 'does not archive' do
-        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
-        expect { subject }.to raise_error('Job is not finished yet')
-        expect(build.trace.exist?).to be_truthy
-      end
-    end
+    it_behaves_like 'trace with enabled live trace feature'
   end
 end

spec/models/ci/build_trace_chunk_spec.rb

+require 'spec_helper'
+
+describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do
+  set(:build) { create(:ci_build, :running) }
+  let(:chunk_index) { 0 }
+  let(:data_store) { :redis }
+  let(:raw_data) { nil }
+
+  let(:build_trace_chunk) do
+    described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
+  end
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
+  describe 'CHUNK_SIZE' do
+    it 'Chunk size can not be changed without special care' do
+      expect(described_class::CHUNK_SIZE).to eq(128.kilobytes)
+    end
+  end
+
+  describe '#data' do
+    subject { build_trace_chunk.data }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, 'Sample data in redis')
+      end
+
+      it { is_expected.to eq('Sample data in redis') }
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+
+      it { is_expected.to eq('Sample data in db') }
+    end
+
+    context 'when data_store is others' do
+      before do
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
+      end
+
+      it { expect { subject }.to raise_error('Unsupported data store') }
+    end
+  end
+
+  describe '#set_data' do
+    subject { build_trace_chunk.send(:set_data, value) }
+
+    let(:value) { 'Sample data' }
+
+    context 'when value bytesize is bigger than CHUNK_SIZE' do
+      let(:value) { 'a' * (described_class::CHUNK_SIZE + 1) }
+
+      it { expect { subject }.to raise_error('too much data') }
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      it do
+        expect(build_trace_chunk.send(:redis_data)).to be_nil
+
+        subject
+
+        expect(build_trace_chunk.send(:redis_data)).to eq(value)
+      end
+
+      context 'when fullfilled chunk size' do
+        let(:value) { 'a' * described_class::CHUNK_SIZE }
+
+        it 'schedules stashing data' do
+          expect(Ci::BuildTraceChunkFlushWorker).to receive(:perform_async).once
+
+          subject
+        end
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      it 'sets data' do
+        expect(build_trace_chunk.raw_data).to be_nil
+
+        subject
+
+        expect(build_trace_chunk.raw_data).to eq(value)
+        expect(build_trace_chunk.persisted?).to be_truthy
+      end
+
+      context 'when raw_data is not changed' do
+        it 'does not execute UPDATE' do
+          expect(build_trace_chunk.raw_data).to be_nil
+          build_trace_chunk.save!
+
+          # First set
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to be > 0
+          expect(build_trace_chunk.raw_data).to eq(value)
+          expect(build_trace_chunk.persisted?).to be_truthy
+
+          # Second set
+          build_trace_chunk.reload
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to be(0)
+        end
+      end
+
+      context 'when fullfilled chunk size' do
+        it 'does not schedule stashing data' do
+          expect(Ci::BuildTraceChunkFlushWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+    end
+
+    context 'when data_store is others' do
+      before do
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
+      end
+
+      it { expect { subject }.to raise_error('Unsupported data store') }
+    end
+  end
+
+  describe '#truncate' do
+    subject { build_trace_chunk.truncate(offset) }
+
+    shared_examples_for 'truncates' do
+      context 'when offset is negative' do
+        let(:offset) { -1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:offset) { data.bytesize + 1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is 10' do
+        let(:offset) { 10 }
+
+        it 'truncates' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset))
+        end
+      end
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+      let(:data) { 'Sample data in redis' }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, data)
+      end
+
+      it_behaves_like 'truncates'
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+      let(:data) { raw_data }
+
+      it_behaves_like 'truncates'
+    end
+  end
+
+  describe '#append' do
+    subject { build_trace_chunk.append(new_data, offset) }
+
+    let(:new_data) { 'Sample new data' }
+    let(:offset) { 0 }
+    let(:total_data) { data + new_data }
+
+    shared_examples_for 'appends' do
+      context 'when offset is negative' do
+        let(:offset) { -1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:offset) { data.bytesize + 1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:new_data) { 'a' * (described_class::CHUNK_SIZE + 1) }
+
+        it { expect { subject }.to raise_error('Chunk size overflow') }
+      end
+
+      context 'when offset is EOF' do
+        let(:offset) { data.bytesize }
+
+        it 'appends' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(total_data)
+        end
+      end
+
+      context 'when offset is 10' do
+        let(:offset) { 10 }
+
+        it 'appends' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
+        end
+      end
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+      let(:data) { 'Sample data in redis' }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, data)
+      end
+
+      it_behaves_like 'appends'
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+      let(:data) { raw_data }
+
+      it_behaves_like 'appends'
+    end
+  end
+
+  describe '#size' do
+    subject { build_trace_chunk.size }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      context 'when data exists' do
+        let(:data) { 'Sample data in redis' }
+
+        before do
+          build_trace_chunk.send(:redis_set_data, data)
+        end
+
+        it { is_expected.to eq(data.bytesize) }
+      end
+
+      context 'when data exists' do
+        it { is_expected.to eq(0) }
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      context 'when data exists' do
+        let(:raw_data) { 'Sample data in db' }
+        let(:data) { raw_data }
+
+        it { is_expected.to eq(data.bytesize) }
+      end
+
+      context 'when data does not exist' do
+        it { is_expected.to eq(0) }
+      end
+    end
+  end
+
+  describe '#use_database!' do
+    subject { build_trace_chunk.use_database! }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      context 'when data exists' do
+        let(:data) { 'Sample data in redis' }
+
+        before do
+          build_trace_chunk.send(:redis_set_data, data)
+        end
+
+        it 'stashes the data' do
+          expect(build_trace_chunk.data_store).to eq('redis')
+          expect(build_trace_chunk.send(:redis_data)).to eq(data)
+          expect(build_trace_chunk.raw_data).to be_nil
+
+          subject
+
+          expect(build_trace_chunk.data_store).to eq('db')
+          expect(build_trace_chunk.send(:redis_data)).to be_nil
+          expect(build_trace_chunk.raw_data).to eq(data)
+        end
+      end
+
+      context 'when data does not exist' do
+        it 'does not call UPDATE' do
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to eq(0)
+        end
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      it 'does not call UPDATE' do
+        expect(ActiveRecord::QueryRecorder.new { subject }.count).to eq(0)
+      end
+    end
+  end
+
+  describe 'ExclusiveLock' do
+    before do
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
+      stub_const('Ci::BuildTraceChunk::WRITE_LOCK_RETRY', 1)
+    end
+
+    it 'raise an error' do
+      expect { build_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
+    end
+  end
+
+  describe 'deletes data in redis after a parent record destroyed' do
+    let(:project) { create(:project) }
+
+    before do
+      pipeline = create(:ci_pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+    end
+
+    shared_examples_for 'deletes all build_trace_chunk and data in redis' do
+      it do
+        Gitlab::Redis::SharedState.with do |redis|
+          expect(redis.scan_each(match: "gitlab:ci:trace:*:chunks:*").to_a.size).to eq(3)
+        end
+
+        expect(described_class.count).to eq(3)
+
+        subject
+
+        expect(described_class.count).to eq(0)
+
+        Gitlab::Redis::SharedState.with do |redis|
+          expect(redis.scan_each(match: "gitlab:ci:trace:*:chunks:*").to_a.size).to eq(0)
+        end
+      end
+    end
+
+    context 'when traces are archived' do
+      let(:subject) do
+        project.builds.each do |build|
+          build.success!
+        end
+      end
+
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
+    end
+
+    context 'when project is destroyed' do
+      let(:subject) do
+        project.destroy!
+      end
+
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
+    end
+  end
+end

spec/requests/api/runner_spec.rb

 require 'spec_helper'
 
-describe API::Runner do
+describe API::Runner, :clean_gitlab_redis_shared_state do
   include StubGitlabCalls
+  include RedisHelpers
 
   let(:registration_token) { 'abcdefg123456' }
 
   before do
+    stub_feature_flags(ci_enable_live_trace: true)
     stub_gitlab_calls
     stub_application_setting(runners_registration_token: registration_token)
     allow_any_instance_of(Ci::Runner).to receive(:cache_attributes)
@@ -882,6 +884,49 @@ describe API::Runner do
             expect(response.status).to eq(403)
           end
         end
+
+        context 'when trace is patched' do
+          before do
+            patch_the_trace
+          end
+
+          it 'has valid trace' do
+            expect(response.status).to eq(202)
+            expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
+          end
+
+          context 'when redis data are flushed' do
+            before do
+              redis_shared_state_cleanup!
+            end
+
+            it 'has empty trace' do
+              expect(job.reload.trace.raw).to eq ''
+            end
+
+            context 'when we perform partial patch' do
+              before do
+                patch_the_trace('hello', headers.merge({ 'Content-Range' => "28-32/5" }))
+              end
+
+              it 'returns an error' do
+                expect(response.status).to eq(416)
+                expect(response.header['Range']).to eq('0-0')
+              end
+            end
+
+            context 'when we resend full trace' do
+              before do
+                patch_the_trace('BUILD TRACE appended appended hello', headers.merge({ 'Content-Range' => "0-34/35" }))
+              end
+
+              it 'succeeds with updating trace' do
+                expect(response.status).to eq(202)
+                expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended hello'
+              end
+            end
+          end
+        end
       end
 
       context 'when Runner makes a force-patch' do
@@ -898,7 +943,7 @@ describe API::Runner do
       end
 
       context 'when content-range start is too big' do
-        let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20' }) }
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20/6' }) }
 
         it 'gets 416 error response with range headers' do
           expect(response.status).to eq 416
@@ -908,7 +953,7 @@ describe API::Runner do
       end
 
       context 'when content-range start is too small' do
-        let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20' }) }
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20/13' }) }
 
         it 'gets 416 error response with range headers' do
           expect(response.status).to eq 416

spec/services/ci/retry_build_service_spec.rb

@@ -32,7 +32,7 @@ describe Ci::RetryBuildService do
        runner_id tag_taggings taggings tags trigger_request_id
        user_id auto_canceled_by_id retried failure_reason
        artifacts_file_store artifacts_metadata_store
-       metadata].freeze
+       metadata trace_chunks].freeze
 
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }

spec/spec_helper.rb

@@ -86,6 +86,7 @@ RSpec.configure do |config|
   config.include WaitForRequests, :js
   config.include LiveDebugger, :js
   config.include MigrationsHelpers, :migration
+  config.include RedisHelpers
 
   if ENV['CI']
     # This includes the first try, i.e. tests will be run 4 times before failing.
@@ -146,21 +147,27 @@ RSpec.configure do |config|
   end
 
   config.around(:each, :clean_gitlab_redis_cache) do |example|
-    Gitlab::Redis::Cache.with(&:flushall)
+    redis_cache_cleanup!
 
     example.run
 
-    Gitlab::Redis::Cache.with(&:flushall)
+    redis_cache_cleanup!
   end
 
   config.around(:each, :clean_gitlab_redis_shared_state) do |example|
-    Gitlab::Redis::SharedState.with(&:flushall)
-    Sidekiq.redis(&:flushall)
+    redis_shared_state_cleanup!
 
     example.run
 
-    Gitlab::Redis::SharedState.with(&:flushall)
-    Sidekiq.redis(&:flushall)
+    redis_shared_state_cleanup!
+  end
+
+  config.around(:each, :clean_gitlab_redis_queues) do |example|
+    redis_queues_cleanup!
+
+    example.run
+
+    redis_queues_cleanup!
   end
 
   # The :each scope runs "inside" the example, so this hook ensures the DB is in the

spec/support/chunked_io/chunked_io_helpers.rb

+module ChunkedIOHelpers
+  def sample_trace_raw
+    @sample_trace_raw ||= File.read(expand_fixture_path('trace/sample_trace'))
+      .force_encoding(Encoding::BINARY)
+  end
+
+  def stub_buffer_size(size)
+    stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size)
+    stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size)
+  end
+end

spec/support/redis/redis_helpers.rb

+module RedisHelpers
+  # config/README.md
+
+  # Usage: performance enhancement
+  def redis_cache_cleanup!
+    Gitlab::Redis::Cache.with(&:flushall)
+  end
+
+  # Usage: SideKiq, Mailroom, CI Runner, Workhorse, push services
+  def redis_queues_cleanup!
+    Gitlab::Redis::Queues.with(&:flushall)
+  end
+
+  # Usage: session state, rate limiting
+  def redis_shared_state_cleanup!
+    Gitlab::Redis::SharedState.with(&:flushall)
+  end
+end

spec/support/shared_examples/ci_trace_shared_examples.rb

+shared_examples_for 'common trace features' do
+  describe '#html' do
+    before do
+      trace.set("12\n34")
+    end
+
+    it "returns formatted html" do
+      expect(trace.html).to eq("12<br>34")
+    end
+
+    it "returns last line of formatted html" do
+      expect(trace.html(last_lines: 1)).to eq("34")
+    end
+  end
+
+  describe '#raw' do
+    before do
+      trace.set("12\n34")
+    end
+
+    it "returns raw output" do
+      expect(trace.raw).to eq("12\n34")
+    end
+
+    it "returns last line of raw output" do
+      expect(trace.raw(last_lines: 1)).to eq("34")
+    end
+  end
+
+  describe '#extract_coverage' do
+    let(:regex) { '\(\d+.\d+\%\) covered' }
+
+    context 'matching coverage' do
+      before do
+        trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
+      end
+
+      it "returns valid coverage" do
+        expect(trace.extract_coverage(regex)).to eq("98.29")
+      end
+    end
+
+    context 'no coverage' do
+      before do
+        trace.set('No coverage')
+      end
+
+      it 'returs nil' do
+        expect(trace.extract_coverage(regex)).to be_nil
+      end
+    end
+  end
+
+  describe '#extract_sections' do
+    let(:log) { 'No sections' }
+    let(:sections) { trace.extract_sections }
+
+    before do
+      trace.set(log)
+    end
+
+    context 'no sections' do
+      it 'returs []' do
+        expect(trace.extract_sections).to eq([])
+      end
+    end
+
+    context 'multiple sections available' do
+      let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) }
+      let(:sections_data) do
+        [
+          { name: 'prepare_script', lines: 2, duration: 3.seconds },
+          { name: 'get_sources', lines: 4, duration: 1.second },
+          { name: 'restore_cache', lines: 0, duration: 0.seconds },
+          { name: 'download_artifacts', lines: 0, duration: 0.seconds },
+          { name: 'build_script', lines: 2, duration: 1.second },
+          { name: 'after_script', lines: 0, duration: 0.seconds },
+          { name: 'archive_cache', lines: 0, duration: 0.seconds },
+          { name: 'upload_artifacts', lines: 0, duration: 0.seconds }
+        ]
+      end
+
+      it "returns valid sections" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(sections_data.size),
+                                 "expected #{sections_data.size} sections, got #{sections.size}"
+
+        buff = StringIO.new(log)
+        sections.each_with_index do |s, i|
+          expected = sections_data[i]
+
+          expect(s[:name]).to eq(expected[:name])
+          expect(s[:date_end] - s[:date_start]).to eq(expected[:duration])
+
+          buff.seek(s[:byte_start], IO::SEEK_SET)
+          length = s[:byte_end] - s[:byte_start]
+          lines = buff.read(length).count("\n")
+          expect(lines).to eq(expected[:lines])
+        end
+      end
+    end
+
+    context 'logs contains "section_start"' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"}
+
+      it "returns only one section" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(1)
+
+        section = sections[0]
+        expect(section[:name]).to eq('a_section')
+        expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section"
+      end
+    end
+
+    context 'missing section_end' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'missing section_start' do
+      let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'inverted section_start section_end' do
+      let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+  end
+
+  describe '#set' do
+    before do
+      trace.set("12")
+    end
+
+    it "returns trace" do
+      expect(trace.raw).to eq("12")
+    end
+
+    context 'overwrite trace' do
+      before do
+        trace.set("34")
+      end
+
+      it "returns new trace" do
+        expect(trace.raw).to eq("34")
+      end
+    end
+
+    context 'runners token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.project.update(runners_token: token)
+        trace.set(token)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+
+    context 'hides build token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.update(token: token)
+        trace.set(token)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+  end
+
+  describe '#append' do
+    before do
+      trace.set("1234")
+    end
+
+    it "returns correct trace" do
+      expect(trace.append("56", 4)).to eq(6)
+      expect(trace.raw).to eq("123456")
+    end
+
+    context 'tries to append trace at different offset' do
+      it "fails with append" do
+        expect(trace.append("56", 2)).to eq(4)
+        expect(trace.raw).to eq("1234")
+      end
+    end
+
+    context 'runners token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.project.update(runners_token: token)
+        trace.append(token, 0)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+
+    context 'build token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.update(token: token)
+        trace.append(token, 0)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+  end
+end
+
+shared_examples_for 'trace with disabled live trace feature' do
+  it_behaves_like 'common trace features'
+
+  describe '#read' do
+    shared_examples 'read successfully with IO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(IO)
+        end
+      end
+    end
+
+    shared_examples 'read successfully with StringIO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(StringIO)
+        end
+      end
+    end
+
+    shared_examples 'failed to read' do
+      it 'yields without source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_nil
+        end
+      end
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_id) exists' do
+      before do
+        expect(trace).to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_ci_id) exists' do
+      before do
+        expect(trace).to receive(:deprecated_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when db trace exists' do
+      before do
+        build.send(:write_attribute, :trace, "data")
+      end
+
+      it_behaves_like 'read successfully with StringIO'
+    end
+
+    context 'when no sources exist' do
+      it_behaves_like 'failed to read'
+    end
+  end
+
+  describe 'trace handling' do
+    subject { trace.exist? }
+
+    context 'trace does not exist' do
+      it { expect(trace.exist?).to be(false) }
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it { is_expected.to be_truthy }
+
+      context 'when the trace artifact has been erased' do
+        before do
+          trace.erase!
+        end
+
+        it { is_expected.to be_falsy }
+
+        it 'removes associations' do
+          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
+        end
+      end
+    end
+
+    context 'new trace path is used' do
+      before do
+        trace.send(:ensure_directory)
+
+        File.open(trace.send(:default_path), "w") do |file|
+          file.write("data")
+        end
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+      end
+    end
+
+    context 'deprecated path' do
+      let(:path) { trace.send(:deprecated_path) }
+
+      context 'with valid ci_id' do
+        before do
+          build.project.update(ci_id: 1000)
+
+          FileUtils.mkdir_p(File.dirname(path))
+
+          File.open(path, "w") do |file|
+            file.write("data")
+          end
+        end
+
+        it "trace exist" do
+          expect(trace.exist?).to be(true)
+        end
+
+        it "can be erased" do
+          trace.erase!
+          expect(trace.exist?).to be(false)
+        end
+      end
+
+      context 'without valid ci_id' do
+        it "does not return deprecated path" do
+          expect(path).to be_nil
+        end
+      end
+    end
+
+    context 'stored in database' do
+      before do
+        build.send(:write_attribute, :trace, "data")
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+      end
+
+      it "returns database data" do
+        expect(trace.raw).to eq("data")
+      end
+    end
+  end
+
+  describe '#archive!' do
+    subject { trace.archive! }
+
+    shared_examples 'archive trace file' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(File.exist?(src_path)).to be_falsy
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace file stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        expect(File.exist?(src_path)).to be_truthy
+      end
+    end
+
+    shared_examples 'archive trace in database' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(build.old_trace).to be_nil
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace in database stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        expect(build.old_trace).to eq(trace_content)
+      end
+    end
+
+    context 'when job does not have trace artifact' do
+      context 'when trace file stored in default path' do
+        let!(:build) { create(:ci_build, :success, :trace_live) }
+        let!(:src_path) { trace.read { |s| s.path } }
+        let!(:src_checksum) { Digest::SHA256.file(src_path).hexdigest }
+
+        it_behaves_like 'archive trace file'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace file stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace file stays intact', error: ActiveRecord::RecordInvalid
+        end
+      end
+
+      context 'when trace is stored in database' do
+        let(:build) { create(:ci_build, :success) }
+        let(:trace_content) { 'Sample trace' }
+        let(:src_checksum) { Digest::SHA256.hexdigest(trace_content) }
+
+        before do
+          build.update_column(:trace, trace_content)
+        end
+
+        it_behaves_like 'archive trace in database'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace in database stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace in database stays intact', error: ActiveRecord::RecordInvalid
+        end
+
+        context 'when there is a validation error on Ci::Build' do
+          before do
+            allow_any_instance_of(Ci::Build).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::Build).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          context "when erase old trace with 'save'" do
+            before do
+              build.send(:write_attribute, :trace, nil)
+              build.save
+            end
+
+            it 'old trace is not deleted' do
+              build.reload
+              expect(build.trace.raw).to eq(trace_content)
+            end
+          end
+
+          it_behaves_like 'archive trace in database'
+        end
+      end
+    end
+
+    context 'when job has trace artifact' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Already archived')
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+      end
+    end
+
+    context 'when job is not finished yet' do
+      let!(:build) { create(:ci_build, :running, :trace_live) }
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Job is not finished yet')
+        expect(build.trace.exist?).to be_truthy
+      end
+    end
+  end
+end
+
+shared_examples_for 'trace with enabled live trace feature' do
+  it_behaves_like 'common trace features'
+
+  describe '#read' do
+    shared_examples 'read successfully with IO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(IO)
+        end
+      end
+    end
+
+    shared_examples 'read successfully with ChunkedIO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(Gitlab::Ci::Trace::ChunkedIO)
+        end
+      end
+    end
+
+    shared_examples 'failed to read' do
+      it 'yields without source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_nil
+        end
+      end
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when live trace exists' do
+      before do
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          stream.write('abc')
+        end
+      end
+
+      it_behaves_like 'read successfully with ChunkedIO'
+    end
+
+    context 'when no sources exist' do
+      it_behaves_like 'failed to read'
+    end
+  end
+
+  describe 'trace handling' do
+    subject { trace.exist? }
+
+    context 'trace does not exist' do
+      it { expect(trace.exist?).to be(false) }
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it { is_expected.to be_truthy }
+
+      context 'when the trace artifact has been erased' do
+        before do
+          trace.erase!
+        end
+
+        it { is_expected.to be_falsy }
+
+        it 'removes associations' do
+          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
+        end
+      end
+    end
+
+    context 'stored in live trace' do
+      before do
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          stream.write('abc')
+        end
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
+      end
+
+      it "returns live trace data" do
+        expect(trace.raw).to eq("abc")
+      end
+    end
+  end
+
+  describe '#archive!' do
+    subject { trace.archive! }
+
+    shared_examples 'archive trace file in ChunkedIO' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace in ChunkedIO stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          expect(stream.read).to eq(trace_raw)
+        end
+      end
+    end
+
+    context 'when job does not have trace artifact' do
+      context 'when trace is stored in ChunkedIO' do
+        let!(:build) { create(:ci_build, :success, :trace_live) }
+        let!(:trace_raw) { build.trace.raw }
+        let!(:src_checksum) { Digest::SHA256.hexdigest(trace_raw) }
+
+        it_behaves_like 'archive trace file in ChunkedIO'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace in ChunkedIO stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace in ChunkedIO stays intact', error: ActiveRecord::RecordInvalid
+        end
+      end
+    end
+
+    context 'when job has trace artifact' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Already archived')
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+      end
+    end
+
+    context 'when job is not finished yet' do
+      let!(:build) { create(:ci_build, :running, :trace_live) }
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Job is not finished yet')
+        expect(build.trace.exist?).to be_truthy
+      end
+    end
+  end
+end