Merge branch 'live-trace-v2-efficient-destroy-all-ee' into 'master'

Port to EE: Live trace: Use efficient destroy all (for `dependent: :destory` problem)

See merge request gitlab-org/gitlab-ee!5593

.flayignore

@@ -16,6 +16,7 @@ lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb
 lib/gitlab/background_migration/*
 app/models/project_services/kubernetes_service.rb
 lib/gitlab/workhorse.rb
+lib/gitlab/ci/trace/chunked_io.rb
 
 ee/db/**/*
 ee/app/serializers/ee/merge_request_widget_entity.rb

app/models/ci/build.rb

@@ -23,6 +23,7 @@ module Ci
 
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
+    has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id

app/models/ci/build_trace_chunk.rb

+module Ci
+  class BuildTraceChunk < ActiveRecord::Base
+    include FastDestroyAll
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
+
+    default_value_for :data_store, :redis
+
+    WriteError = Class.new(StandardError)
+
+    CHUNK_SIZE = 128.kilobytes
+    CHUNK_REDIS_TTL = 1.week
+    WRITE_LOCK_RETRY = 10
+    WRITE_LOCK_SLEEP = 0.01.seconds
+    WRITE_LOCK_TTL = 1.minute
+
+    enum data_store: {
+      redis: 1,
+      db: 2
+    }
+
+    class << self
+      def redis_data_key(build_id, chunk_index)
+        "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}"
+      end
+
+      def redis_data_keys
+        redis.pluck(:build_id, :chunk_index).map do |data|
+          redis_data_key(data.first, data.second)
+        end
+      end
+
+      def redis_delete_data(keys)
+        return if keys.empty?
+
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.del(keys)
+        end
+      end
+
+      ##
+      # FastDestroyAll concerns
+      def begin_fast_destroy
+        redis_data_keys
+      end
+
+      ##
+      # FastDestroyAll concerns
+      def finalize_fast_destroy(keys)
+        redis_delete_data(keys)
+      end
+    end
+
+    ##
+    # Data is memoized for optimizing #size and #end_offset
+    def data
+      @data ||= get_data.to_s
+    end
+
+    def truncate(offset = 0)
+      raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
+      return if offset == size # Skip the following process as it doesn't affect anything
+
+      self.append("", offset)
+    end
+
+    def append(new_data, offset)
+      raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
+      raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize)
+
+      set_data(data.byteslice(0, offset) + new_data)
+    end
+
+    def size
+      data&.bytesize.to_i
+    end
+
+    def start_offset
+      chunk_index * CHUNK_SIZE
+    end
+
+    def end_offset
+      start_offset + size
+    end
+
+    def range
+      (start_offset...end_offset)
+    end
+
+    def use_database!
+      in_lock do
+        break if db?
+        break unless size > 0
+
+        self.update!(raw_data: data, data_store: :db)
+        self.class.redis_delete_data([redis_data_key])
+      end
+    end
+
+    private
+
+    def get_data
+      if redis?
+        redis_data
+      elsif db?
+        raw_data
+      else
+        raise 'Unsupported data store'
+      end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default
+    end
+
+    def set_data(value)
+      raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE
+
+      in_lock do
+        if redis?
+          redis_set_data(value)
+        elsif db?
+          self.raw_data = value
+        else
+          raise 'Unsupported data store'
+        end
+
+        @data = value
+
+        save! if changed?
+      end
+
+      schedule_to_db if full?
+    end
+
+    def schedule_to_db
+      return if db?
+
+      Ci::BuildTraceChunkFlushWorker.perform_async(id)
+    end
+
+    def full?
+      size == CHUNK_SIZE
+    end
+
+    def redis_data
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.get(redis_data_key)
+      end
+    end
+
+    def redis_set_data(data)
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL)
+      end
+    end
+
+    def redis_data_key
+      self.class.redis_data_key(build_id, chunk_index)
+    end
+
+    def in_lock
+      write_lock_key = "trace_write:#{build_id}:chunks:#{chunk_index}"
+
+      lease = Gitlab::ExclusiveLease.new(write_lock_key, timeout: WRITE_LOCK_TTL)
+      retry_count = 0
+
+      until uuid = lease.try_obtain
+        # Keep trying until we obtain the lease. To prevent hammering Redis too
+        # much we'll wait for a bit between retries.
+        sleep(WRITE_LOCK_SLEEP)
+        break if WRITE_LOCK_RETRY < (retry_count += 1)
+      end
+
+      raise WriteError, 'Failed to obtain write lock' unless uuid
+
+      self.reload if self.persisted?
+      return yield
+    ensure
+      Gitlab::ExclusiveLease.cancel(write_lock_key, uuid)
+    end
+  end
+end

app/models/concerns/fast_destroy_all.rb

+##
+# This module is for replacing `dependent: :destroy` and `before_destroy` hooks.
+#
+# In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas,
+# `delete_all` is efficient as it deletes all rows with a single `DELETE` query.
+#
+# It's better to use `delete_all` as our best practice, however,
+# if external data (e.g. ObjectStorage, FileStorage or Redis) are assosiated with database records,
+# it is difficult to accomplish it.
+#
+# This module defines a format to use `delete_all` and delete associated external data.
+# Here is an exmaple
+#
+# Situation
+# - `Project` has many `Ci::BuildTraceChunk` through `Ci::Build`
+# - `Ci::BuildTraceChunk` stores associated data in Redis, so it relies on `dependent: :destroy` and `before_destroy` for the deletion
+#
+# How to use
+# - Define `use_fast_destroy :build_trace_chunks` in `Project` model.
+# - Define `begin_fast_destroy` and `finalize_fast_destroy(params)` in `Ci::BuildTraceChunk` model.
+# - Use `fast_destroy_all` instead of `destroy` and `destroy_all`
+# - Remove `dependent: :destroy` and `before_destroy` as it's no longer need
+#
+# Expectation
+# - When a project is `destroy`ed, the associated trace_chunks will be deleted by `delete_all`,
+#   and the associated data will be removed, too.
+# - When `fast_destroy_all` is called, it also performns as same.
+module FastDestroyAll
+  extend ActiveSupport::Concern
+
+  ForbiddenActionError = Class.new(StandardError)
+
+  included do
+    before_destroy do
+      raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`'
+    end
+  end
+
+  class_methods do
+    ##
+    # This method delete rows and associated external data efficiently
+    #
+    # This method can replace `destroy` and `destroy_all` without having `after_destroy` hook
+    def fast_destroy_all
+      params = begin_fast_destroy
+
+      delete_all
+
+      finalize_fast_destroy(params)
+    end
+
+    ##
+    # This method returns identifiers to delete associated external data (e.g. file paths, redis keys)
+    #
+    # This method must be defined in fast destroyable model
+    def begin_fast_destroy
+      raise NotImplementedError
+    end
+
+    ##
+    # This method deletes associated external data with the identifiers returned by `begin_fast_destroy`
+    #
+    # This method must be defined in fast destroyable model
+    def finalize_fast_destroy(params)
+      raise NotImplementedError
+    end
+  end
+
+  module Helpers
+    extend ActiveSupport::Concern
+
+    class_methods do
+      ##
+      # This method is to be defined on models which have fast destroyable models as children,
+      # and let us avoid to use `dependent: :destroy` hook
+      def use_fast_destroy(relation)
+        before_destroy(prepend: true) do
+          perform_fast_destroy(public_send(relation)) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+
+    def perform_fast_destroy(subject)
+      params = subject.begin_fast_destroy
+
+      run_after_commit do
+        subject.finalize_fast_destroy(params)
+      end
+    end
+  end
+end

app/models/project.rb

@@ -22,6 +22,7 @@ class Project < ActiveRecord::Base
   include DeploymentPlatform
   include ::Gitlab::Utils::StrongMemoize
   include ChronicDurationAttribute
+  include FastDestroyAll::Helpers
 
   # EE specific modules
   prepend EE::Project
@@ -86,6 +87,9 @@ class Project < ActiveRecord::Base
   after_update :update_forks_visibility_level
 
   before_destroy :remove_private_deploy_keys
+
+  use_fast_destroy :build_trace_chunks
+
   after_destroy -> { run_after_commit { remove_pages } }
   after_destroy :remove_exports
 
@@ -233,6 +237,7 @@ class Project < ActiveRecord::Base
   # still using `dependent: :destroy` here.
   has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
+  has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks
   has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'

app/workers/all_queues.yml

@@ -52,6 +52,7 @@
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
 - pipeline_background:archive_trace
+- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
 - pipeline_default:pipeline_metrics

app/workers/ci/build_trace_chunk_flush_worker.rb

+module Ci
+  class BuildTraceChunkFlushWorker
+    include ApplicationWorker
+    include PipelineBackgroundQueue
+
+    def perform(build_trace_chunk_id)
+      ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|
+        build_trace_chunk.use_database!
+      end
+    end
+  end
+end

changelogs/unreleased/live-trace-v2-efficient-destroy-all.yml

+---
+title: Destroy build_chunks efficiently with FastDestroyAll module
+merge_request: 18575
+author:
+type: performance

changelogs/unreleased/live-trace-v2.yml

+---
+title: New CI Job live-trace architecture
+merge_request: 18169
+author:
+type: changed

db/migrate/20180326202229_create_ci_build_trace_chunks.rb

+class CreateCiBuildTraceChunks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_build_trace_chunks, id: :bigserial do |t|
+      t.integer :build_id, null: false
+      t.integer :chunk_index, null: false
+      t.integer :data_store, null: false
+      t.binary :raw_data
+
+      t.foreign_key :ci_builds, column: :build_id, on_delete: :cascade
+      t.index [:build_id, :chunk_index], unique: true
+    end
+  end
+end

db/migrate/20180406204716_add_limits_ci_build_trace_chunks_raw_data_for_mysql.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
+
+class AddLimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    LimitsCiBuildTraceChunksRawDataForMysql.new.up
+  end
+end

db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql.rb

+class LimitsCiBuildTraceChunksRawDataForMysql < ActiveRecord::Migration
+  def up
+    return unless Gitlab::Database.mysql?
+
+    # Mysql needs MEDIUMTEXT type (up to 16MB) rather than TEXT (up to 64KB)
+    # Because 'raw_data' is always capped by Ci::BuildTraceChunk::CHUNK_SIZE, which is 128KB
+    change_column :ci_build_trace_chunks, :raw_data, :binary, limit: 16.megabytes - 1 #MEDIUMTEXT
+  end
+end

db/schema.rb

@@ -342,6 +342,15 @@ ActiveRecord::Schema.define(version: 20180503154922) do
 
   add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
 
+  create_table "ci_build_trace_chunks", id: :bigserial, force: :cascade do |t|
+    t.integer "build_id", null: false
+    t.integer "chunk_index", null: false
+    t.integer "data_store", null: false
+    t.binary "raw_data"
+  end
+
+  add_index "ci_build_trace_chunks", ["build_id", "chunk_index"], name: "index_ci_build_trace_chunks_on_build_id_and_chunk_index", unique: true, using: :btree
+
   create_table "ci_build_trace_section_names", force: :cascade do |t|
     t.integer "project_id", null: false
     t.string "name", null: false
@@ -2723,6 +2732,7 @@ ActiveRecord::Schema.define(version: 20180503154922) do
   add_foreign_key "boards", "namespaces", column: "group_id", name: "fk_1e9a074a35", on_delete: :cascade
   add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
+  add_foreign_key "ci_build_trace_chunks", "ci_builds", column: "build_id", on_delete: :cascade
   add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade

doc/administration/job_traces.md

@@ -40,3 +40,98 @@ To change the location where the job logs will be stored, follow the steps below
 
 [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
 [restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab"
+
+## New live trace architecture
+
+> [Introduced][ce-18169] in GitLab 10.4.
+
+> **Notes**:
+- This feature is still Beta, which could impact GitLab.com/on-premises instances, and in the worst case scenario, traces will be lost.
+- This feature is still being discussed in [an issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/46097) for the performance improvements.
+- This feature is off by default. Please check below how to enable/disable this featrue.
+
+**What is "live trace"?**
+
+Job trace that is sent by runner while jobs are running. You can see live trace in job pages UI.
+The live traces are archived once job finishes.
+
+**What is new architecture?**
+
+So far, when GitLab Runner sends a job trace to GitLab-Rails, traces have been saved to file storage as text files.
+This was a problem for [Cloud Native-compatible GitLab application](https://gitlab.com/gitlab-com/migration/issues/23) where GitLab had to rely on File Storage.
+
+This new live trace architecture stores chunks of traces in Redis and database instead of file storage.
+Redis is used as first-class storage, and it stores up-to 128kB. Once the full chunk is sent it will be flushed to database. Afterwhile, the data in Redis and database will be archived to ObjectStorage.
+
+Here is the detailed data flow.
+
+1. GitLab Runner picks a job from GitLab-Rails
+1. GitLab Runner sends a piece of trace to GitLab-Rails
+1. GitLab-Rails appends the data to Redis
+1. If the data in Redis is fulfilled 128kB, the data is flushed to Database.
+1. 2.~4. is continued until the job is finished
+1. Once the job is finished, GitLab-Rails schedules a sidekiq worker to archive the trace
+1. The sidekiq worker archives the trace to Object Storage, and cleanup the trace in Redis and Database
+
+**How to check if it's on or off?**
+
+```ruby
+Feature.enabled?('ci_enable_live_trace')
+```
+
+**How to enable?**
+
+```ruby
+Feature.enable('ci_enable_live_trace')
+```
+
+>**Note:**
+The transition period will be handled gracefully. Upcoming traces will be generated with the new architecture, and on-going live traces will stay with the legacy architecture (i.e. on-going live traces won't be re-generated forcibly with the new architecture).
+
+**How to disable?**
+
+```ruby
+Feature.disable('ci_enable_live_trace')
+```
+
+>**Note:**
+The transition period will be handled gracefully. Upcoming traces will be generated with the legacy architecture, and on-going live traces will stay with the new architecture (i.e. on-going live traces won't be re-generated forcibly with the legacy architecture).
+
+**Redis namespace:**
+
+`Gitlab::Redis::SharedState`
+
+**Potential impact:**
+
+- This feature could incur data loss:
+  - Case 1: When all data in Redis are accidentally flushed.
+    - On-going live traces could be recovered by re-sending traces (This is supported by all versions of GitLab Runner)
+    - Finished jobs which has not archived live traces will lose the last part (~128kB) of trace data.
+  - Case 2: When sidekiq workers failed to archive (e.g. There was a bug that prevents archiving process, Sidekiq inconsistancy, etc):
+    - Currently all trace data in Redis will be deleted after one week. If the sidekiq workers can't finish by the expiry date, the part of trace data will be lost.
+- This feature could consume all memory on Redis instance. If the number of jobs is 1000, 128MB (128kB * 1000) is consumed.
+- This feature could pressure Database replication lag. `INSERT` are generated to indicate that we have trace chunk. `UPDATE` with 128kB of data is issued once we receive multiple chunks.
+- and so on
+
+**How to test?**
+
+We're currently evaluating this feature on dev.gitalb.org or staging.gitlab.com to verify this features. Here is the list of tests/measurements.
+
+- Features:
+  - Live traces should be visible on job pages
+  - Archived traces should be visible on job pages
+  - Live traces should be archived to Object storage
+  - Live traces should be cleaned up after archived
+  - etc
+- Performance:
+  - Schedule 1000~10000 jobs and let GitLab-runners process concurrently. Measure memoery presssure, IO load, etc.
+  - etc
+- Failover:
+  - Simulate Redis outage
+  - etc
+
+**How to verify the correctnesss?**
+
+- TBD
+
+[ce-44935]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18169

lib/api/runner.rb

@@ -153,9 +153,20 @@ module API
         content_range = request.headers['Content-Range']
         content_range = content_range.split('-')
 
-        stream_size = job.trace.append(request.body.read, content_range[0].to_i)
-        if stream_size < 0
-          break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
+        # TODO:
+        # it seems that `Content-Range` as formatted by runner is wrong,
+        # the `byte_end` should point to final byte, but it points byte+1
+        # that means that we have to calculate end of body,
+        # as we cannot use `content_length[1]`
+        # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275
+
+        body_data = request.body.read
+        body_start = content_range[0].to_i
+        body_end = body_start + body_data.bytesize
+
+        stream_size = job.trace.append(body_data, body_start)
+        unless stream_size == body_end
+          break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{stream_size}" })
         end
 
         status 202

lib/gitlab/ci/trace.rb

@@ -36,16 +36,16 @@ module Gitlab
       end
 
       def set(data)
-        write do |stream|
+        write('w+b') do |stream|
           data = job.hide_secrets(data)
           stream.set(data)
         end
       end
 
       def append(data, offset)
-        write do |stream|
+        write('a+b') do |stream|
           current_length = stream.size
-          break -current_length unless current_length == offset
+          break current_length unless current_length == offset
 
           data = job.hide_secrets(data)
           stream.append(data, offset)
@@ -54,13 +54,15 @@ module Gitlab
       end
 
       def exist?
-        trace_artifact&.exists? || current_path.present? || old_trace.present?
+        trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
           if trace_artifact
             trace_artifact.open
+          elsif job.trace_chunks.any?
+            Gitlab::Ci::Trace::ChunkedIO.new(job)
           elsif current_path
             File.open(current_path, "rb")
           elsif old_trace
@@ -73,9 +75,15 @@ module Gitlab
         stream&.close
       end
 
-      def write
+      def write(mode)
         stream = Gitlab::Ci::Trace::Stream.new do
-          File.open(ensure_path, "a+b")
+          if current_path
+            File.open(current_path, mode)
+          elsif Feature.enabled?('ci_enable_live_trace')
+            Gitlab::Ci::Trace::ChunkedIO.new(job)
+          else
+            File.open(ensure_path, mode)
+          end
         end
 
         yield(stream).tap do
@@ -92,6 +100,7 @@ module Gitlab
           FileUtils.rm(trace_path, force: true)
         end
 
+        job.trace_chunks.fast_destroy_all
         job.erase_old_trace!
       end
 
@@ -99,7 +108,12 @@ module Gitlab
         raise ArchiveError, 'Already archived' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        if current_path
+        if job.trace_chunks.any?
+          Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
+            archive_stream!(stream)
+            stream.destroy!
+          end
+        elsif current_path
           File.open(current_path) do |stream|
             archive_stream!(stream)
             FileUtils.rm(current_path)
@@ -116,7 +130,7 @@ module Gitlab
 
       def archive_stream!(stream)
         clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
-          create_job_trace!(job, clone_path)
+          create_build_trace!(job, clone_path)
         end
       end
 
@@ -132,7 +146,7 @@ module Gitlab
         end
       end
 
-      def create_job_trace!(job, path)
+      def create_build_trace!(job, path)
         File.open(path) do |stream|
           job.create_job_artifacts_trace!(
             project: job.project,

lib/gitlab/ci/trace/chunked_io.rb

+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  module Ci
+    class Trace
+      class ChunkedIO
+        CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE
+
+        FailedToGetChunkError = Class.new(StandardError)
+
+        attr_reader :build
+        attr_reader :tell, :size
+        attr_reader :chunk, :chunk_range
+
+        alias_method :pos, :tell
+
+        def initialize(build, &block)
+          @build = build
+          @chunks_cache = []
+          @tell = 0
+          @size = calculate_size
+          yield self if block_given?
+        end
+
+        def close
+          # no-op
+        end
+
+        def binmode
+          # no-op
+        end
+
+        def binmode?
+          true
+        end
+
+        def seek(pos, where = IO::SEEK_SET)
+          new_pos =
+            case where
+            when IO::SEEK_END
+              size + pos
+            when IO::SEEK_SET
+              pos
+            when IO::SEEK_CUR
+              tell + pos
+            else
+              -1
+            end
+
+          raise ArgumentError, 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+          @tell = new_pos
+        end
+
+        def eof?
+          tell == size
+        end
+
+        def each_line
+          until eof?
+            line = readline
+            break if line.nil?
+
+            yield(line)
+          end
+        end
+
+        def read(length = nil, outbuf = "")
+          out = ""
+
+          length ||= size - tell
+
+          until length <= 0 || eof?
+            data = chunk_slice_from_offset
+            break if data.empty?
+
+            chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min
+            chunk_data = data.byteslice(0, chunk_bytes)
+
+            out << chunk_data
+            @tell += chunk_data.bytesize
+            length -= chunk_data.bytesize
+          end
+
+          # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality
+          if outbuf
+            outbuf.slice!(0, outbuf.bytesize)
+            outbuf << out
+          end
+
+          out
+        end
+
+        def readline
+          out = ""
+
+          until eof?
+            data = chunk_slice_from_offset
+            new_line = data.index("\n")
+
+            if !new_line.nil?
+              out << data[0..new_line]
+              @tell += new_line + 1
+              break
+            else
+              out << data
+              @tell += data.bytesize
+            end
+          end
+
+          out
+        end
+
+        def write(data)
+          start_pos = tell
+
+          while tell < start_pos + data.bytesize
+            # get slice from current offset till the end where it falls into chunk
+            chunk_bytes = CHUNK_SIZE - chunk_offset
+            chunk_data = data.byteslice(tell - start_pos, chunk_bytes)
+
+            # append data to chunk, overwriting from that point
+            ensure_chunk.append(chunk_data, chunk_offset)
+
+            # move offsets within buffer
+            @tell += chunk_data.bytesize
+            @size = [size, tell].max
+          end
+
+          tell - start_pos
+        ensure
+          invalidate_chunk_cache
+        end
+
+        def truncate(offset)
+          raise ArgumentError, 'Outside of file' if offset > size || offset < 0
+          return if offset == size # Skip the following process as it doesn't affect anything
+
+          @tell = offset
+          @size = offset
+
+          # remove all next chunks
+          trace_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
+
+          # truncate current chunk
+          current_chunk.truncate(chunk_offset)
+        ensure
+          invalidate_chunk_cache
+        end
+
+        def flush
+          # no-op
+        end
+
+        def present?
+          true
+        end
+
+        def destroy!
+          trace_chunks.fast_destroy_all
+          @tell = @size = 0
+        ensure
+          invalidate_chunk_cache
+        end
+
+        private
+
+        ##
+        # The below methods are not implemented in IO class
+        #
+        def in_range?
+          @chunk_range&.include?(tell)
+        end
+
+        def chunk_slice_from_offset
+          unless in_range?
+            current_chunk.tap do |chunk|
+              raise FailedToGetChunkError unless chunk
+
+              @chunk = chunk.data
+              @chunk_range = chunk.range
+            end
+          end
+
+          @chunk[chunk_offset..CHUNK_SIZE]
+        end
+
+        def chunk_offset
+          tell % CHUNK_SIZE
+        end
+
+        def chunk_index
+          tell / CHUNK_SIZE
+        end
+
+        def chunk_start
+          chunk_index * CHUNK_SIZE
+        end
+
+        def chunk_end
+          [chunk_start + CHUNK_SIZE, size].min
+        end
+
+        def invalidate_chunk_cache
+          @chunks_cache = []
+        end
+
+        def current_chunk
+          @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
+        end
+
+        def build_chunk
+          @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
+        end
+
+        def ensure_chunk
+          current_chunk || build_chunk
+        end
+
+        def trace_chunks
+          ::Ci::BuildTraceChunk.where(build: build)
+        end
+
+        def calculate_size
+          trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/trace/stream.rb

@@ -39,6 +39,8 @@ module Gitlab
         end
 
         def append(data, offset)
+          data = data.force_encoding(Encoding::BINARY)
+
           stream.truncate(offset)
           stream.seek(0, IO::SEEK_END)
           stream.write(data)
@@ -46,8 +48,11 @@ module Gitlab
         end
 
         def set(data)
-          truncate(0)
+          data = data.force_encoding(Encoding::BINARY)
+
+          stream.seek(0, IO::SEEK_SET)
           stream.write(data)
+          stream.truncate(data.bytesize)
           stream.flush()
         end
 
@@ -127,11 +132,11 @@ module Gitlab
             buf += debris
             debris, *lines = buf.each_line.to_a
             lines.reverse_each do |line|
-              yield(line.force_encoding('UTF-8'))
+              yield(line.force_encoding(Encoding.default_external))
             end
           end
 
-          yield(debris.force_encoding('UTF-8')) unless debris.empty?
+          yield(debris.force_encoding(Encoding.default_external)) unless debris.empty?
         end
 
         def read_backward(length)

lib/tasks/migrate/add_limits_mysql.rake

 require Rails.root.join('db/migrate/limits_to_mysql')
 require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -8,4 +9,5 @@ task add_limits_mysql: :environment do
   LimitsToMysql.new.up
   MarkdownCacheLimitsToMysql.new.up
   MergeRequestDiffFileLimitsToMysql.new.up
+  LimitsCiBuildTraceChunksRawDataForMysql.new.up
 end

spec/controllers/projects/jobs_controller_spec.rb

 # coding: utf-8
 require 'spec_helper'
 
-describe Projects::JobsController do
+describe Projects::JobsController, :clean_gitlab_redis_shared_state do
   include ApiHelpers
   include HttpIOHelpers
 
@@ -10,6 +10,7 @@ describe Projects::JobsController do
   let(:user) { create(:user) }
 
   before do
+    stub_feature_flags(ci_enable_live_trace: true)
     stub_not_protect_default_branch
   end
 

spec/factories/ci/build_trace_chunks.rb

+FactoryBot.define do
+  factory :ci_build_trace_chunk, class: Ci::BuildTraceChunk do
+    build factory: :ci_build
+    chunk_index 0
+    data_store :redis
+  end
+end

spec/features/projects/jobs_spec.rb

 require 'spec_helper'
 require 'tempfile'
 
-feature 'Jobs' do
+feature 'Jobs', :clean_gitlab_redis_shared_state do
   let(:user) { create(:user) }
   let(:user_access_level) { :developer }
   let(:project) { create(:project, :repository) }
@@ -282,7 +282,7 @@ feature 'Jobs' do
         it 'loads job trace' do
           expect(page).to have_content 'BUILD TRACE'
 
-          job.trace.write do |stream|
+          job.trace.write('a+b') do |stream|
             stream.append(' and more trace', 11)
           end
 
@@ -604,48 +604,6 @@ feature 'Jobs' do
       end
     end
 
-    context 'storage form' do
-      let(:existing_file) { Tempfile.new('existing-trace-file').path }
-
-      before do
-        job.run!
-      end
-
-      context 'when job has trace in file', :js do
-        let(:paths) do
-          [existing_file]
-        end
-
-        before do
-          allow_any_instance_of(Gitlab::Ci::Trace)
-            .to receive(:paths)
-            .and_return([existing_file])
-        end
-
-        it 'sends the right headers' do
-          requests = inspect_requests(inject_headers: { 'X-Sendfile-Type' => 'X-Sendfile' }) do
-            visit raw_project_job_path(project, job)
-          end
-          expect(requests.first.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
-          expect(requests.first.response_headers['X-Sendfile']).to eq(existing_file)
-        end
-      end
-
-      context 'when job has trace in the database', :js do
-        before do
-          allow_any_instance_of(Gitlab::Ci::Trace)
-            .to receive(:paths)
-            .and_return([])
-
-          visit project_job_path(project, job)
-        end
-
-        it 'sends the right headers' do
-          expect(page).not_to have_selector('.js-raw-link-controller')
-        end
-      end
-    end
-
     context "when visiting old URL" do
       let(:raw_job_url) do
         raw_project_job_path(project, job)

spec/lib/gitlab/ci/trace/chunked_io_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do
+  include ChunkedIOHelpers
+
+  set(:build) { create(:ci_build, :running) }
+  let(:chunked_io) { described_class.new(build) }
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
+  context "#initialize" do
+    context 'when a chunk exists' do
+      before do
+        build.trace.set('ABC')
+      end
+
+      it { expect(chunked_io.size).to eq(3) }
+    end
+
+    context 'when two chunks exist' do
+      before do
+        stub_buffer_size(4)
+        build.trace.set('ABCDEF')
+      end
+
+      it { expect(chunked_io.size).to eq(6) }
+    end
+
+    context 'when no chunks exists' do
+      it { expect(chunked_io.size).to eq(0) }
+    end
+  end
+
+  context "#seek" do
+    subject { chunked_io.seek(pos, where) }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    context 'when moves pos to end of the file' do
+      let(:pos) { 0 }
+      let(:where) { IO::SEEK_END }
+
+      it { is_expected.to eq(sample_trace_raw.bytesize) }
+    end
+
+    context 'when moves pos to middle of the file' do
+      let(:pos) { sample_trace_raw.bytesize / 2 }
+      let(:where) { IO::SEEK_SET }
+
+      it { is_expected.to eq(pos) }
+    end
+
+    context 'when moves pos around' do
+      it 'matches the result' do
+        expect(chunked_io.seek(0)).to eq(0)
+        expect(chunked_io.seek(100, IO::SEEK_CUR)).to eq(100)
+        expect { chunked_io.seek(sample_trace_raw.bytesize + 1, IO::SEEK_CUR) }
+          .to raise_error('new position is outside of file')
+      end
+    end
+  end
+
+  context "#eof?" do
+    subject { chunked_io.eof? }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    context 'when current pos is at end of the file' do
+      before do
+        chunked_io.seek(sample_trace_raw.bytesize, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when current pos is not at end of the file' do
+      before do
+        chunked_io.seek(0, IO::SEEK_SET)
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  context "#each_line" do
+    let(:string_io) { StringIO.new(sample_trace_raw) }
+
+    context 'when buffer size is smaller than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it 'yields lines' do
+        expect { |b| chunked_io.each_line(&b) }
+          .to yield_successive_args(*string_io.each_line.to_a)
+      end
+    end
+
+    context 'when buffer size is larger than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize * 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it 'calls get_chunk only once' do
+        expect_any_instance_of(Gitlab::Ci::Trace::ChunkedIO)
+          .to receive(:current_chunk).once.and_call_original
+
+        chunked_io.each_line { |line| }
+      end
+    end
+  end
+
+  context "#read" do
+    subject { chunked_io.read(length) }
+
+    context 'when read the whole size' do
+      let(:length) { nil }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it { is_expected.to eq(sample_trace_raw) }
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it { is_expected.to eq(sample_trace_raw) }
+      end
+    end
+
+    context 'when read only first 100 bytes' do
+      let(:length) { 100 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw.byteslice(0, length))
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw.byteslice(0, length))
+        end
+      end
+    end
+
+    context 'when tries to read oversize' do
+      let(:length) { sample_trace_raw.bytesize + 1000 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw)
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to eq(sample_trace_raw)
+        end
+      end
+    end
+
+    context 'when tries to read 0 bytes' do
+      let(:length) { 0 }
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to be_empty
+        end
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it 'reads a trace' do
+          is_expected.to be_empty
+        end
+      end
+    end
+  end
+
+  context "#readline" do
+    subject { chunked_io.readline }
+
+    let(:string_io) { StringIO.new(sample_trace_raw) }
+
+    shared_examples 'all line matching' do
+      it do
+        (0...sample_trace_raw.lines.count).each do
+          expect(chunked_io.readline).to eq(string_io.readline)
+        end
+      end
+    end
+
+    context 'when buffer size is smaller than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when buffer size is larger than file size' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize * 2)
+        build.trace.set(sample_trace_raw)
+      end
+
+      it_behaves_like 'all line matching'
+    end
+
+    context 'when pos is at middle of the file' do
+      before do
+        stub_buffer_size(sample_trace_raw.bytesize / 2)
+        build.trace.set(sample_trace_raw)
+
+        chunked_io.seek(chunked_io.size / 2)
+        string_io.seek(string_io.size / 2)
+      end
+
+      it 'reads from pos' do
+        expect(chunked_io.readline).to eq(string_io.readline)
+      end
+    end
+  end
+
+  context "#write" do
+    subject { chunked_io.write(data) }
+
+    let(:data) { sample_trace_raw }
+
+    context 'when data does not exist' do
+      shared_examples 'writes a trace' do
+        it do
+          is_expected.to eq(data.bytesize)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(data)
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(data.bytesize / 2)
+        end
+
+        it_behaves_like 'writes a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(data.bytesize * 2)
+        end
+
+        it_behaves_like 'writes a trace'
+      end
+    end
+
+    context 'when data already exists' do
+      let(:exist_data) { 'exist data' }
+
+      shared_examples 'appends a trace' do
+        it do
+          chunked_io.seek(0, IO::SEEK_END)
+          is_expected.to eq(data.bytesize)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(exist_data + data)
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(exist_data)
+        end
+
+        it_behaves_like 'appends a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(exist_data)
+        end
+
+        it_behaves_like 'appends a trace'
+      end
+    end
+  end
+
+  context "#truncate" do
+    let(:offset) { 10 }
+
+    context 'when data does not exist' do
+      shared_examples 'truncates a trace' do
+        it do
+          chunked_io.truncate(offset)
+
+          chunked_io.seek(0, IO::SEEK_SET)
+          expect(chunked_io.read).to eq(sample_trace_raw.byteslice(0, offset))
+        end
+      end
+
+      context 'when buffer size is smaller than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize / 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it_behaves_like 'truncates a trace'
+      end
+
+      context 'when buffer size is larger than file size' do
+        before do
+          stub_buffer_size(sample_trace_raw.bytesize * 2)
+          build.trace.set(sample_trace_raw)
+        end
+
+        it_behaves_like 'truncates a trace'
+      end
+    end
+  end
+
+  context "#destroy!" do
+    subject { chunked_io.destroy! }
+
+    before do
+      build.trace.set(sample_trace_raw)
+    end
+
+    it 'deletes' do
+      expect { subject }.to change { chunked_io.size }
+        .from(sample_trace_raw.bytesize).to(0)
+
+      expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0)
+    end
+  end
+end

spec/lib/gitlab/ci/trace/stream_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Ci::Trace::Stream do
+describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do
+  set(:build) { create(:ci_build, :running) }
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
   describe 'delegates' do
     subject { described_class.new { nil } }
 
@@ -11,337 +17,470 @@ describe Gitlab::Ci::Trace::Stream do
     it { is_expected.to delegate_method(:path).to(:stream) }
     it { is_expected.to delegate_method(:truncate).to(:stream) }
     it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) }
-    it { is_expected.to delegate_method(:file?).to(:path).as(:present?) }
   end
 
   describe '#limit' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new((1..8).to_a.join("\n"))
+    shared_examples_for 'limits' do
+      it 'if size is larger we start from beginning' do
+        stream.limit(20)
+
+        expect(stream.tell).to eq(0)
       end
-    end
 
-    it 'if size is larger we start from beginning' do
-      stream.limit(20)
+      it 'if size is smaller we start from the end' do
+        stream.limit(2)
 
-      expect(stream.tell).to eq(0)
-    end
+        expect(stream.raw).to eq("8")
+      end
 
-    it 'if size is smaller we start from the end' do
-      stream.limit(2)
+      context 'when the trace contains ANSI sequence and Unicode' do
+        let(:stream) do
+          described_class.new do
+            File.open(expand_fixture_path('trace/ansi-sequence-and-unicode'))
+          end
+        end
 
-      expect(stream.raw).to eq("8")
-    end
+        it 'forwards to the next linefeed, case 1' do
+          stream.limit(7)
 
-    context 'when the trace contains ANSI sequence and Unicode' do
-      let(:stream) do
-        described_class.new do
-          File.open(expand_fixture_path('trace/ansi-sequence-and-unicode'))
+          result = stream.raw
+
+          expect(result).to eq('')
+          expect(result.encoding).to eq(Encoding.default_external)
         end
-      end
 
-      it 'forwards to the next linefeed, case 1' do
-        stream.limit(7)
+        it 'forwards to the next linefeed, case 2' do
+          stream.limit(29)
 
-        result = stream.raw
+          result = stream.raw
 
-        expect(result).to eq('')
-        expect(result.encoding).to eq(Encoding.default_external)
-      end
+          expect(result).to eq("\e[01;32m許功蓋\e[0m\n")
+          expect(result.encoding).to eq(Encoding.default_external)
+        end
 
-      it 'forwards to the next linefeed, case 2' do
-        stream.limit(29)
+        # See https://gitlab.com/gitlab-org/gitlab-ce/issues/30796
+        it 'reads in binary, output as Encoding.default_external' do
+          stream.limit(52)
 
-        result = stream.raw
+          result = stream.html
 
-        expect(result).to eq("\e[01;32m許功蓋\e[0m\n")
-        expect(result.encoding).to eq(Encoding.default_external)
+          expect(result).to eq("ヾ(´༎ຶД༎ຶ`)ﾉ<br><span class=\"term-fg-green\">許功蓋</span><br>")
+          expect(result.encoding).to eq(Encoding.default_external)
+        end
       end
+    end
 
-      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/30796
-      it 'reads in binary, output as Encoding.default_external' do
-        stream.limit(52)
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new((1..8).to_a.join("\n"))
+        end
+      end
 
-        result = stream.html
+      it_behaves_like 'limits'
+    end
 
-        expect(result).to eq("ヾ(´༎ຶД༎ຶ`)ﾉ<br><span class=\"term-fg-green\">許功蓋</span><br>")
-        expect(result.encoding).to eq(Encoding.default_external)
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write((1..8).to_a.join("\n"))
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
       end
+
+      it_behaves_like 'limits'
     end
   end
 
   describe '#append' do
-    let(:tempfile) { Tempfile.new }
+    shared_examples_for 'appends' do
+      it "truncates and append content" do
+        stream.append("89", 4)
+        stream.seek(0)
 
-    let(:stream) do
-      described_class.new do
-        tempfile.write("12345678")
-        tempfile.rewind
-        tempfile
+        expect(stream.size).to eq(6)
+        expect(stream.raw).to eq("123489")
       end
-    end
 
-    after do
-      tempfile.unlink
-    end
+      it 'appends in binary mode' do
+        '😺'.force_encoding('ASCII-8BIT').each_char.with_index do |byte, offset|
+          stream.append(byte, offset)
+        end
 
-    it "truncates and append content" do
-      stream.append("89", 4)
-      stream.seek(0)
+        stream.seek(0)
 
-      expect(stream.size).to eq(6)
-      expect(stream.raw).to eq("123489")
+        expect(stream.size).to eq(4)
+        expect(stream.raw).to eq('😺')
+      end
     end
 
-    it 'appends in binary mode' do
-      '😺'.force_encoding('ASCII-8BIT').each_char.with_index do |byte, offset|
-        stream.append(byte, offset)
+    context 'when stream is Tempfile' do
+      let(:tempfile) { Tempfile.new }
+
+      let(:stream) do
+        described_class.new do
+          tempfile.write("12345678")
+          tempfile.rewind
+          tempfile
+        end
+      end
+
+      after do
+        tempfile.unlink
       end
 
-      stream.seek(0)
+      it_behaves_like 'appends'
+    end
 
-      expect(stream.size).to eq(4)
-      expect(stream.raw).to eq('😺')
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write('12345678')
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'appends'
     end
   end
 
   describe '#set' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new("12345678")
+    shared_examples_for 'sets' do
+      before do
+        stream.set("8901")
+      end
+
+      it "overwrite content" do
+        stream.seek(0)
+
+        expect(stream.size).to eq(4)
+        expect(stream.raw).to eq("8901")
       end
     end
 
-    before do
-      stream.set("8901")
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new("12345678")
+        end
+      end
+
+      it_behaves_like 'sets'
     end
 
-    it "overwrite content" do
-      stream.seek(0)
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write('12345678')
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
 
-      expect(stream.size).to eq(4)
-      expect(stream.raw).to eq("8901")
+      it_behaves_like 'sets'
     end
   end
 
   describe '#raw' do
-    let(:path) { __FILE__ }
-    let(:lines) { File.readlines(path) }
-    let(:stream) do
-      described_class.new do
-        File.open(path)
+    shared_examples_for 'sets' do
+      it 'returns all contents if last_lines is not specified' do
+        result = stream.raw
+
+        expect(result).to eq(lines.join)
+        expect(result.encoding).to eq(Encoding.default_external)
       end
-    end
 
-    it 'returns all contents if last_lines is not specified' do
-      result = stream.raw
+      context 'limit max lines' do
+        before do
+          # specifying BUFFER_SIZE forces to seek backwards
+          allow(described_class).to receive(:BUFFER_SIZE)
+            .and_return(2)
+        end
 
-      expect(result).to eq(lines.join)
-      expect(result.encoding).to eq(Encoding.default_external)
-    end
+        it 'returns last few lines' do
+          result = stream.raw(last_lines: 2)
 
-    context 'limit max lines' do
-      before do
-        # specifying BUFFER_SIZE forces to seek backwards
-        allow(described_class).to receive(:BUFFER_SIZE)
-          .and_return(2)
+          expect(result).to eq(lines.last(2).join)
+          expect(result.encoding).to eq(Encoding.default_external)
+        end
+
+        it 'returns everything if trying to get too many lines' do
+          result = stream.raw(last_lines: lines.size * 2)
+
+          expect(result).to eq(lines.join)
+          expect(result.encoding).to eq(Encoding.default_external)
+        end
       end
+    end
 
-      it 'returns last few lines' do
-        result = stream.raw(last_lines: 2)
+    let(:path) { __FILE__ }
+    let(:lines) { File.readlines(path) }
 
-        expect(result).to eq(lines.last(2).join)
-        expect(result.encoding).to eq(Encoding.default_external)
+    context 'when stream is File' do
+      let(:stream) do
+        described_class.new do
+          File.open(path)
+        end
       end
 
-      it 'returns everything if trying to get too many lines' do
-        result = stream.raw(last_lines: lines.size * 2)
+      it_behaves_like 'sets'
+    end
 
-        expect(result).to eq(lines.join)
-        expect(result.encoding).to eq(Encoding.default_external)
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write(File.binread(path))
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
       end
+
+      it_behaves_like 'sets'
     end
   end
 
   describe '#html_with_state' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new("1234")
+    shared_examples_for 'html_with_states' do
+      it 'returns html content with state' do
+        result = stream.html_with_state
+
+        expect(result.html).to eq("1234")
       end
-    end
 
-    it 'returns html content with state' do
-      result = stream.html_with_state
+      context 'follow-up state' do
+        let!(:last_result) { stream.html_with_state }
 
-      expect(result.html).to eq("1234")
-    end
+        before do
+          stream.append("5678", 4)
+          stream.seek(0)
+        end
 
-    context 'follow-up state' do
-      let!(:last_result) { stream.html_with_state }
+        it "returns appended trace" do
+          result = stream.html_with_state(last_result.state)
 
-      before do
-        stream.append("5678", 4)
-        stream.seek(0)
+          expect(result.append).to be_truthy
+          expect(result.html).to eq("5678")
+        end
+      end
+    end
+
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new("1234")
+        end
       end
 
-      it "returns appended trace" do
-        result = stream.html_with_state(last_result.state)
+      it_behaves_like 'html_with_states'
+    end
 
-        expect(result.append).to be_truthy
-        expect(result.html).to eq("5678")
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write("1234")
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
       end
+
+      it_behaves_like 'html_with_states'
     end
   end
 
   describe '#html' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new("12\n34\n56")
+    shared_examples_for 'htmls' do
+      it "returns html" do
+        expect(stream.html).to eq("12<br>34<br>56")
+      end
+
+      it "returns html for last line only" do
+        expect(stream.html(last_lines: 1)).to eq("56")
       end
     end
 
-    it "returns html" do
-      expect(stream.html).to eq("12<br>34<br>56")
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new("12\n34\n56")
+        end
+      end
+
+      it_behaves_like 'htmls'
     end
 
-    it "returns html for last line only" do
-      expect(stream.html(last_lines: 1)).to eq("56")
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write("12\n34\n56")
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
+      end
+
+      it_behaves_like 'htmls'
     end
   end
 
   describe '#extract_coverage' do
-    let(:stream) do
-      described_class.new do
-        StringIO.new(data)
-      end
-    end
+    shared_examples_for 'extract_coverages' do
+      context 'valid content & regex' do
+        let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' }
+        let(:regex) { '\(\d+.\d+\%\) covered' }
 
-    subject { stream.extract_coverage(regex) }
+        it { is_expected.to eq("98.29") }
+      end
 
-    context 'valid content & regex' do
-      let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' }
-      let(:regex) { '\(\d+.\d+\%\) covered' }
+      context 'valid content & bad regex' do
+        let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
+        let(:regex) { 'very covered' }
 
-      it { is_expected.to eq("98.29") }
-    end
+        it { is_expected.to be_nil }
+      end
 
-    context 'valid content & bad regex' do
-      let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
-      let(:regex) { 'very covered' }
+      context 'no coverage content & regex' do
+        let(:data) { 'No coverage for today :sad:' }
+        let(:regex) { '\(\d+.\d+\%\) covered' }
 
-      it { is_expected.to be_nil }
-    end
+        it { is_expected.to be_nil }
+      end
 
-    context 'no coverage content & regex' do
-      let(:data) { 'No coverage for today :sad:' }
-      let(:regex) { '\(\d+.\d+\%\) covered' }
+      context 'multiple results in content & regex' do
+        let(:data) do
+          <<~HEREDOC
+            (98.39%) covered
+            (98.29%) covered
+          HEREDOC
+        end
 
-      it { is_expected.to be_nil }
-    end
+        let(:regex) { '\(\d+.\d+\%\) covered' }
 
-    context 'multiple results in content & regex' do
-      let(:data) do
-        <<~HEREDOC
-          (98.39%) covered
-          (98.29%) covered
-        HEREDOC
+        it 'returns the last matched coverage' do
+          is_expected.to eq("98.29")
+        end
       end
 
-      let(:regex) { '\(\d+.\d+\%\) covered' }
+      context 'when BUFFER_SIZE is smaller than stream.size' do
+        let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
+        let(:regex) { '\(\d+.\d+\%\) covered' }
 
-      it 'returns the last matched coverage' do
-        is_expected.to eq("98.29")
+        before do
+          stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5)
+        end
+
+        it { is_expected.to eq("98.29") }
       end
-    end
 
-    context 'when BUFFER_SIZE is smaller than stream.size' do
-      let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
-      let(:regex) { '\(\d+.\d+\%\) covered' }
+      context 'when regex is multi-byte char' do
+        let(:data) { '95.0 ゴッドファット\n' }
+        let(:regex) { '\d+\.\d+ ゴッドファット' }
 
-      before do
-        stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5)
+        before do
+          stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5)
+        end
+
+        it { is_expected.to eq('95.0') }
       end
 
-      it { is_expected.to eq("98.29") }
-    end
+      context 'when BUFFER_SIZE is equal to stream.size' do
+        let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
+        let(:regex) { '\(\d+.\d+\%\) covered' }
 
-    context 'when regex is multi-byte char' do
-      let(:data) { '95.0 ゴッドファット\n' }
-      let(:regex) { '\d+\.\d+ ゴッドファット' }
+        before do
+          stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', data.length)
+        end
 
-      before do
-        stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5)
+        it { is_expected.to eq("98.29") }
       end
 
-      it { is_expected.to eq('95.0') }
-    end
-
-    context 'when BUFFER_SIZE is equal to stream.size' do
-      let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
-      let(:regex) { '\(\d+.\d+\%\) covered' }
+      context 'using a regex capture' do
+        let(:data) { 'TOTAL      9926   3489    65%' }
+        let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' }
 
-      before do
-        stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', data.length)
+        it { is_expected.to eq("65") }
       end
 
-      it { is_expected.to eq("98.29") }
-    end
+      context 'malicious regexp' do
+        let(:data) { malicious_text }
+        let(:regex) { malicious_regexp }
 
-    context 'using a regex capture' do
-      let(:data) { 'TOTAL      9926   3489    65%' }
-      let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' }
+        include_examples 'malicious regexp'
+      end
 
-      it { is_expected.to eq("65") }
-    end
+      context 'multi-line data with rooted regexp' do
+        let(:data) { "\n65%\n" }
+        let(:regex) { '^(\d+)\%$' }
 
-    context 'malicious regexp' do
-      let(:data) { malicious_text }
-      let(:regex) { malicious_regexp }
+        it { is_expected.to eq('65') }
+      end
 
-      include_examples 'malicious regexp'
-    end
+      context 'long line' do
+        let(:data) { 'a' * 80000 + '100%' + 'a' * 80000 }
+        let(:regex) { '\d+\%' }
 
-    context 'multi-line data with rooted regexp' do
-      let(:data) { "\n65%\n" }
-      let(:regex) { '^(\d+)\%$' }
+        it { is_expected.to eq('100') }
+      end
 
-      it { is_expected.to eq('65') }
-    end
+      context 'many lines' do
+        let(:data) { "foo\n" * 80000 + "100%\n" + "foo\n" * 80000 }
+        let(:regex) { '\d+\%' }
 
-    context 'long line' do
-      let(:data) { 'a' * 80000 + '100%' + 'a' * 80000 }
-      let(:regex) { '\d+\%' }
+        it { is_expected.to eq('100') }
+      end
 
-      it { is_expected.to eq('100') }
-    end
+      context 'empty regex' do
+        let(:data) { 'foo' }
+        let(:regex) { '' }
 
-    context 'many lines' do
-      let(:data) { "foo\n" * 80000 + "100%\n" + "foo\n" * 80000 }
-      let(:regex) { '\d+\%' }
+        it 'skips processing' do
+          expect(stream).not_to receive(:read)
 
-      it { is_expected.to eq('100') }
-    end
+          is_expected.to be_nil
+        end
+      end
 
-    context 'empty regex' do
-      let(:data) { 'foo' }
-      let(:regex) { '' }
+      context 'nil regex' do
+        let(:data) { 'foo' }
+        let(:regex) { nil }
 
-      it 'skips processing' do
-        expect(stream).not_to receive(:read)
+        it 'skips processing' do
+          expect(stream).not_to receive(:read)
 
-        is_expected.to be_nil
+          is_expected.to be_nil
+        end
       end
     end
 
-    context 'nil regex' do
-      let(:data) { 'foo' }
-      let(:regex) { nil }
+    subject { stream.extract_coverage(regex) }
 
-      it 'skips processing' do
-        expect(stream).not_to receive(:read)
+    context 'when stream is StringIO' do
+      let(:stream) do
+        described_class.new do
+          StringIO.new(data)
+        end
+      end
+
+      it_behaves_like 'extract_coverages'
+    end
 
-        is_expected.to be_nil
+    context 'when stream is ChunkedIO' do
+      let(:stream) do
+        described_class.new do
+          Gitlab::Ci::Trace::ChunkedIO.new(build).tap do |chunked_io|
+            chunked_io.write(data)
+            chunked_io.seek(0, IO::SEEK_SET)
+          end
+        end
       end
+
+      it_behaves_like 'extract_coverages'
     end
   end
 end

spec/lib/gitlab/ci/trace_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Ci::Trace do
+describe Gitlab::Ci::Trace, :clean_gitlab_redis_cache do
   let(:build) { create(:ci_build) }
   let(:trace) { described_class.new(build) }
 
@@ -9,552 +9,19 @@ describe Gitlab::Ci::Trace do
     it { expect(trace).to delegate_method(:old_trace).to(:job) }
   end
 
-  describe '#html' do
+  context 'when live trace feature is disabled' do
     before do
-      trace.set("12\n34")
+      stub_feature_flags(ci_enable_live_trace: false)
     end
 
-    it "returns formatted html" do
-      expect(trace.html).to eq("12<br>34")
-    end
-
-    it "returns last line of formatted html" do
-      expect(trace.html(last_lines: 1)).to eq("34")
-    end
-  end
-
-  describe '#raw' do
-    before do
-      trace.set("12\n34")
-    end
-
-    it "returns raw output" do
-      expect(trace.raw).to eq("12\n34")
-    end
-
-    it "returns last line of raw output" do
-      expect(trace.raw(last_lines: 1)).to eq("34")
-    end
-  end
-
-  describe '#extract_coverage' do
-    let(:regex) { '\(\d+.\d+\%\) covered' }
-
-    context 'matching coverage' do
-      before do
-        trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
-      end
-
-      it "returns valid coverage" do
-        expect(trace.extract_coverage(regex)).to eq("98.29")
-      end
-    end
-
-    context 'no coverage' do
-      before do
-        trace.set('No coverage')
-      end
-
-      it 'returs nil' do
-        expect(trace.extract_coverage(regex)).to be_nil
-      end
-    end
-  end
-
-  describe '#extract_sections' do
-    let(:log) { 'No sections' }
-    let(:sections) { trace.extract_sections }
-
-    before do
-      trace.set(log)
-    end
-
-    context 'no sections' do
-      it 'returs []' do
-        expect(trace.extract_sections).to eq([])
-      end
-    end
-
-    context 'multiple sections available' do
-      let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) }
-      let(:sections_data) do
-        [
-          { name: 'prepare_script', lines: 2, duration: 3.seconds },
-          { name: 'get_sources', lines: 4, duration: 1.second },
-          { name: 'restore_cache', lines: 0, duration: 0.seconds },
-          { name: 'download_artifacts', lines: 0, duration: 0.seconds },
-          { name: 'build_script', lines: 2, duration: 1.second },
-          { name: 'after_script', lines: 0, duration: 0.seconds },
-          { name: 'archive_cache', lines: 0, duration: 0.seconds },
-          { name: 'upload_artifacts', lines: 0, duration: 0.seconds }
-        ]
-      end
-
-      it "returns valid sections" do
-        expect(sections).not_to be_empty
-        expect(sections.size).to eq(sections_data.size),
-                                 "expected #{sections_data.size} sections, got #{sections.size}"
-
-        buff = StringIO.new(log)
-        sections.each_with_index do |s, i|
-          expected = sections_data[i]
-
-          expect(s[:name]).to eq(expected[:name])
-          expect(s[:date_end] - s[:date_start]).to eq(expected[:duration])
-
-          buff.seek(s[:byte_start], IO::SEEK_SET)
-          length = s[:byte_end] - s[:byte_start]
-          lines = buff.read(length).count("\n")
-          expect(lines).to eq(expected[:lines])
-        end
-      end
-    end
-
-    context 'logs contains "section_start"' do
-      let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"}
-
-      it "returns only one section" do
-        expect(sections).not_to be_empty
-        expect(sections.size).to eq(1)
-
-        section = sections[0]
-        expect(section[:name]).to eq('a_section')
-        expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section"
-      end
-    end
-
-    context 'missing section_end' do
-      let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-
-    context 'missing section_start' do
-      let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-
-    context 'inverted section_start section_end' do
-      let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"}
-
-      it "returns no sections" do
-        expect(sections).to be_empty
-      end
-    end
-  end
-
-  describe '#set' do
-    before do
-      trace.set("12")
-    end
-
-    it "returns trace" do
-      expect(trace.raw).to eq("12")
-    end
-
-    context 'overwrite trace' do
-      before do
-        trace.set("34")
-      end
-
-      it "returns new trace" do
-        expect(trace.raw).to eq("34")
-      end
-    end
-
-    context 'runners token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.project.update(runners_token: token)
-        trace.set(token)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-
-    context 'hides build token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.update(token: token)
-        trace.set(token)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
+    it_behaves_like 'trace with disabled live trace feature'
   end
 
-  describe '#append' do
+  context 'when live trace feature is enabled' do
     before do
-      trace.set("1234")
-    end
-
-    it "returns correct trace" do
-      expect(trace.append("56", 4)).to eq(6)
-      expect(trace.raw).to eq("123456")
-    end
-
-    context 'tries to append trace at different offset' do
-      it "fails with append" do
-        expect(trace.append("56", 2)).to eq(-4)
-        expect(trace.raw).to eq("1234")
-      end
-    end
-
-    context 'runners token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.project.update(runners_token: token)
-        trace.append(token, 0)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-
-    context 'build token' do
-      let(:token) { 'my_secret_token' }
-
-      before do
-        build.update(token: token)
-        trace.append(token, 0)
-      end
-
-      it "hides token" do
-        expect(trace.raw).not_to include(token)
-      end
-    end
-  end
-
-  describe '#read' do
-    shared_examples 'read successfully with IO' do
-      it 'yields with source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_a(IO)
-        end
-      end
-    end
-
-    shared_examples 'read successfully with StringIO' do
-      it 'yields with source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_a(StringIO)
-        end
-      end
-    end
-
-    shared_examples 'failed to read' do
-      it 'yields without source' do
-        trace.read do |stream|
-          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
-          expect(stream.stream).to be_nil
-        end
-      end
-    end
-
-    context 'when trace artifact exists' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when current_path (with project_id) exists' do
-      before do
-        expect(trace).to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
-      end
-
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when current_path (with project_ci_id) exists' do
-      before do
-        expect(trace).to receive(:deprecated_path) { expand_fixture_path('trace/sample_trace') }
-      end
-
-      it_behaves_like 'read successfully with IO'
-    end
-
-    context 'when db trace exists' do
-      before do
-        build.send(:write_attribute, :trace, "data")
-      end
-
-      it_behaves_like 'read successfully with StringIO'
-    end
-
-    context 'when no sources exist' do
-      it_behaves_like 'failed to read'
-    end
-  end
-
-  describe 'trace handling' do
-    subject { trace.exist? }
-
-    context 'trace does not exist' do
-      it { expect(trace.exist?).to be(false) }
-    end
-
-    context 'when trace artifact exists' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it { is_expected.to be_truthy }
-
-      context 'when the trace artifact has been erased' do
-        before do
-          trace.erase!
-        end
-
-        it { is_expected.to be_falsy }
-
-        it 'removes associations' do
-          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
-        end
-      end
-    end
-
-    context 'new trace path is used' do
-      before do
-        trace.send(:ensure_directory)
-
-        File.open(trace.send(:default_path), "w") do |file|
-          file.write("data")
-        end
-      end
-
-      it "trace exist" do
-        expect(trace.exist?).to be(true)
-      end
-
-      it "can be erased" do
-        trace.erase!
-        expect(trace.exist?).to be(false)
-      end
-    end
-
-    context 'deprecated path' do
-      let(:path) { trace.send(:deprecated_path) }
-
-      context 'with valid ci_id' do
-        before do
-          build.project.update(ci_id: 1000)
-
-          FileUtils.mkdir_p(File.dirname(path))
-
-          File.open(path, "w") do |file|
-            file.write("data")
-          end
-        end
-
-        it "trace exist" do
-          expect(trace.exist?).to be(true)
-        end
-
-        it "can be erased" do
-          trace.erase!
-          expect(trace.exist?).to be(false)
-        end
-      end
-
-      context 'without valid ci_id' do
-        it "does not return deprecated path" do
-          expect(path).to be_nil
-        end
-      end
-    end
-
-    context 'stored in database' do
-      before do
-        build.send(:write_attribute, :trace, "data")
-      end
-
-      it "trace exist" do
-        expect(trace.exist?).to be(true)
-      end
-
-      it "can be erased" do
-        trace.erase!
-        expect(trace.exist?).to be(false)
-      end
-
-      it "returns database data" do
-        expect(trace.raw).to eq("data")
-      end
-    end
-  end
-
-  describe '#archive!' do
-    subject { trace.archive! }
-
-    shared_examples 'archive trace file' do
-      it do
-        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
-        expect(File.exist?(src_path)).to be_falsy
-        expect(src_checksum)
-          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
-        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
-      end
-    end
-
-    shared_examples 'source trace file stays intact' do |error:|
-      it do
-        expect { subject }.to raise_error(error)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace).to be_nil
-        expect(File.exist?(src_path)).to be_truthy
-      end
-    end
-
-    shared_examples 'archive trace in database' do
-      it do
-        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
-        expect(build.old_trace).to be_nil
-        expect(src_checksum)
-          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
-        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
-      end
-    end
-
-    shared_examples 'source trace in database stays intact' do |error:|
-      it do
-        expect { subject }.to raise_error(error)
-
-        build.reload
-        expect(build.trace.exist?).to be_truthy
-        expect(build.job_artifacts_trace).to be_nil
-        expect(build.old_trace).to eq(trace_content)
-      end
-    end
-
-    context 'when job does not have trace artifact' do
-      context 'when trace file stored in default path' do
-        let!(:build) { create(:ci_build, :success, :trace_live) }
-        let!(:src_path) { trace.read { |s| s.path } }
-        let!(:src_checksum) { Digest::SHA256.file(src_path).hexdigest }
-
-        it_behaves_like 'archive trace file'
-
-        context 'when failed to create clone file' do
-          before do
-            allow(IO).to receive(:copy_stream).and_return(0)
-          end
-
-          it_behaves_like 'source trace file stays intact', error: Gitlab::Ci::Trace::ArchiveError
-        end
-
-        context 'when failed to create job artifact record' do
-          before do
-            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          it_behaves_like 'source trace file stays intact', error: ActiveRecord::RecordInvalid
-        end
-      end
-
-      context 'when trace is stored in database' do
-        let(:build) { create(:ci_build, :success) }
-        let(:trace_content) { 'Sample trace' }
-        let!(:src_checksum) { Digest::SHA256.hexdigest(trace_content) }
-
-        before do
-          build.update_column(:trace, trace_content)
-        end
-
-        it_behaves_like 'archive trace in database'
-
-        context 'when failed to create clone file' do
-          before do
-            allow(IO).to receive(:copy_stream).and_return(0)
-          end
-
-          it_behaves_like 'source trace in database stays intact', error: Gitlab::Ci::Trace::ArchiveError
-        end
-
-        context 'when failed to create job artifact record' do
-          before do
-            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          it_behaves_like 'source trace in database stays intact', error: ActiveRecord::RecordInvalid
-        end
-
-        context 'when there is a validation error on Ci::Build' do
-          before do
-            allow_any_instance_of(Ci::Build).to receive(:save).and_return(false)
-            allow_any_instance_of(Ci::Build).to receive_message_chain(:errors, :full_messages)
-              .and_return(%w[Error Error])
-          end
-
-          context "when erase old trace with 'save'" do
-            before do
-              build.send(:write_attribute, :trace, nil)
-              build.save
-            end
-
-            it 'old trace is not deleted' do
-              build.reload
-              expect(build.trace.raw).to eq(trace_content)
-            end
-          end
-
-          it_behaves_like 'archive trace in database'
-        end
-      end
+      stub_feature_flags(ci_enable_live_trace: true)
     end
 
-    context 'when job has trace artifact' do
-      before do
-        create(:ci_job_artifact, :trace, job: build)
-      end
-
-      it 'does not archive' do
-        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
-        expect { subject }.to raise_error('Already archived')
-        expect(build.job_artifacts_trace.file.exists?).to be_truthy
-      end
-    end
-
-    context 'when job is not finished yet' do
-      let!(:build) { create(:ci_build, :running, :trace_live) }
-
-      it 'does not archive' do
-        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
-        expect { subject }.to raise_error('Job is not finished yet')
-        expect(build.trace.exist?).to be_truthy
-      end
-    end
+    it_behaves_like 'trace with enabled live trace feature'
   end
 end

spec/lib/gitlab/import_export/all_models.yml

@@ -313,6 +313,7 @@ project:
 - sourced_pipelines
 - members_and_requesters
 - build_trace_section_names
+- build_trace_chunks
 - root_of_fork_network
 - fork_network_member
 - fork_network

spec/models/ci/build_trace_chunk_spec.rb

+require 'spec_helper'
+
+describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do
+  set(:build) { create(:ci_build, :running) }
+  let(:chunk_index) { 0 }
+  let(:data_store) { :redis }
+  let(:raw_data) { nil }
+
+  let(:build_trace_chunk) do
+    described_class.new(build: build, chunk_index: chunk_index, data_store: data_store, raw_data: raw_data)
+  end
+
+  before do
+    stub_feature_flags(ci_enable_live_trace: true)
+  end
+
+  context 'FastDestroyAll' do
+    let(:parent) { create(:project) }
+    let(:pipeline) { create(:ci_pipeline, project: parent) }
+    let(:build) { create(:ci_build, :running, :trace_live, pipeline: pipeline, project: parent) }
+    let(:subjects) { build.trace_chunks }
+
+    it_behaves_like 'fast destroyable'
+
+    def external_data_counter
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.scan_each(match: "gitlab:ci:trace:*:chunks:*").to_a.size
+      end
+    end
+  end
+
+  describe 'CHUNK_SIZE' do
+    it 'Chunk size can not be changed without special care' do
+      expect(described_class::CHUNK_SIZE).to eq(128.kilobytes)
+    end
+  end
+
+  describe '#data' do
+    subject { build_trace_chunk.data }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, 'Sample data in redis')
+      end
+
+      it { is_expected.to eq('Sample data in redis') }
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+
+      it { is_expected.to eq('Sample data in db') }
+    end
+
+    context 'when data_store is others' do
+      before do
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
+      end
+
+      it { expect { subject }.to raise_error('Unsupported data store') }
+    end
+  end
+
+  describe '#set_data' do
+    subject { build_trace_chunk.send(:set_data, value) }
+
+    let(:value) { 'Sample data' }
+
+    context 'when value bytesize is bigger than CHUNK_SIZE' do
+      let(:value) { 'a' * (described_class::CHUNK_SIZE + 1) }
+
+      it { expect { subject }.to raise_error('too much data') }
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      it do
+        expect(build_trace_chunk.send(:redis_data)).to be_nil
+
+        subject
+
+        expect(build_trace_chunk.send(:redis_data)).to eq(value)
+      end
+
+      context 'when fullfilled chunk size' do
+        let(:value) { 'a' * described_class::CHUNK_SIZE }
+
+        it 'schedules stashing data' do
+          expect(Ci::BuildTraceChunkFlushWorker).to receive(:perform_async).once
+
+          subject
+        end
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      it 'sets data' do
+        expect(build_trace_chunk.raw_data).to be_nil
+
+        subject
+
+        expect(build_trace_chunk.raw_data).to eq(value)
+        expect(build_trace_chunk.persisted?).to be_truthy
+      end
+
+      context 'when raw_data is not changed' do
+        it 'does not execute UPDATE' do
+          expect(build_trace_chunk.raw_data).to be_nil
+          build_trace_chunk.save!
+
+          # First set
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to be > 0
+          expect(build_trace_chunk.raw_data).to eq(value)
+          expect(build_trace_chunk.persisted?).to be_truthy
+
+          # Second set
+          build_trace_chunk.reload
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to be(0)
+        end
+      end
+
+      context 'when fullfilled chunk size' do
+        it 'does not schedule stashing data' do
+          expect(Ci::BuildTraceChunkFlushWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
+    end
+
+    context 'when data_store is others' do
+      before do
+        build_trace_chunk.send(:write_attribute, :data_store, -1)
+      end
+
+      it { expect { subject }.to raise_error('Unsupported data store') }
+    end
+  end
+
+  describe '#truncate' do
+    subject { build_trace_chunk.truncate(offset) }
+
+    shared_examples_for 'truncates' do
+      context 'when offset is negative' do
+        let(:offset) { -1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:offset) { data.bytesize + 1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is 10' do
+        let(:offset) { 10 }
+
+        it 'truncates' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset))
+        end
+      end
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+      let(:data) { 'Sample data in redis' }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, data)
+      end
+
+      it_behaves_like 'truncates'
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+      let(:data) { raw_data }
+
+      it_behaves_like 'truncates'
+    end
+  end
+
+  describe '#append' do
+    subject { build_trace_chunk.append(new_data, offset) }
+
+    let(:new_data) { 'Sample new data' }
+    let(:offset) { 0 }
+    let(:total_data) { data + new_data }
+
+    shared_examples_for 'appends' do
+      context 'when offset is negative' do
+        let(:offset) { -1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:offset) { data.bytesize + 1 }
+
+        it { expect { subject }.to raise_error('Offset is out of range') }
+      end
+
+      context 'when offset is bigger than data size' do
+        let(:new_data) { 'a' * (described_class::CHUNK_SIZE + 1) }
+
+        it { expect { subject }.to raise_error('Chunk size overflow') }
+      end
+
+      context 'when offset is EOF' do
+        let(:offset) { data.bytesize }
+
+        it 'appends' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(total_data)
+        end
+      end
+
+      context 'when offset is 10' do
+        let(:offset) { 10 }
+
+        it 'appends' do
+          subject
+
+          expect(build_trace_chunk.data).to eq(data.byteslice(0, offset) + new_data)
+        end
+      end
+    end
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+      let(:data) { 'Sample data in redis' }
+
+      before do
+        build_trace_chunk.send(:redis_set_data, data)
+      end
+
+      it_behaves_like 'appends'
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+      let(:raw_data) { 'Sample data in db' }
+      let(:data) { raw_data }
+
+      it_behaves_like 'appends'
+    end
+  end
+
+  describe '#size' do
+    subject { build_trace_chunk.size }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      context 'when data exists' do
+        let(:data) { 'Sample data in redis' }
+
+        before do
+          build_trace_chunk.send(:redis_set_data, data)
+        end
+
+        it { is_expected.to eq(data.bytesize) }
+      end
+
+      context 'when data exists' do
+        it { is_expected.to eq(0) }
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      context 'when data exists' do
+        let(:raw_data) { 'Sample data in db' }
+        let(:data) { raw_data }
+
+        it { is_expected.to eq(data.bytesize) }
+      end
+
+      context 'when data does not exist' do
+        it { is_expected.to eq(0) }
+      end
+    end
+  end
+
+  describe '#use_database!' do
+    subject { build_trace_chunk.use_database! }
+
+    context 'when data_store is redis' do
+      let(:data_store) { :redis }
+
+      context 'when data exists' do
+        let(:data) { 'Sample data in redis' }
+
+        before do
+          build_trace_chunk.send(:redis_set_data, data)
+        end
+
+        it 'stashes the data' do
+          expect(build_trace_chunk.data_store).to eq('redis')
+          expect(build_trace_chunk.send(:redis_data)).to eq(data)
+          expect(build_trace_chunk.raw_data).to be_nil
+
+          subject
+
+          expect(build_trace_chunk.data_store).to eq('db')
+          expect(build_trace_chunk.send(:redis_data)).to be_nil
+          expect(build_trace_chunk.raw_data).to eq(data)
+        end
+      end
+
+      context 'when data does not exist' do
+        it 'does not call UPDATE' do
+          expect(ActiveRecord::QueryRecorder.new { subject }.count).to eq(0)
+        end
+      end
+    end
+
+    context 'when data_store is database' do
+      let(:data_store) { :db }
+
+      it 'does not call UPDATE' do
+        expect(ActiveRecord::QueryRecorder.new { subject }.count).to eq(0)
+      end
+    end
+  end
+
+  describe 'ExclusiveLock' do
+    before do
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
+      stub_const('Ci::BuildTraceChunk::WRITE_LOCK_RETRY', 1)
+    end
+
+    it 'raise an error' do
+      expect { build_trace_chunk.append('ABC', 0) }.to raise_error('Failed to obtain write lock')
+    end
+  end
+
+  describe 'deletes data in redis after a parent record destroyed' do
+    let(:project) { create(:project) }
+
+    before do
+      pipeline = create(:ci_pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+      create(:ci_build, :running, :trace_live, pipeline: pipeline, project: project)
+    end
+
+    shared_examples_for 'deletes all build_trace_chunk and data in redis' do
+      it do
+        Gitlab::Redis::SharedState.with do |redis|
+          expect(redis.scan_each(match: "gitlab:ci:trace:*:chunks:*").to_a.size).to eq(3)
+        end
+
+        expect(described_class.count).to eq(3)
+
+        subject
+
+        expect(described_class.count).to eq(0)
+
+        Gitlab::Redis::SharedState.with do |redis|
+          expect(redis.scan_each(match: "gitlab:ci:trace:*:chunks:*").to_a.size).to eq(0)
+        end
+      end
+    end
+
+    context 'when traces are archived' do
+      let(:subject) do
+        project.builds.each do |build|
+          build.success!
+        end
+      end
+
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
+    end
+
+    context 'when project is destroyed' do
+      let(:subject) do
+        project.destroy!
+      end
+
+      it_behaves_like 'deletes all build_trace_chunk and data in redis'
+    end
+  end
+end

spec/requests/api/runner_spec.rb

 require 'spec_helper'
 
-describe API::Runner do
+describe API::Runner, :clean_gitlab_redis_shared_state do
   include StubGitlabCalls
+  include RedisHelpers
 
   let(:registration_token) { 'abcdefg123456' }
 
   before do
+    stub_feature_flags(ci_enable_live_trace: true)
     stub_gitlab_calls
     stub_application_setting(runners_registration_token: registration_token)
     allow_any_instance_of(Ci::Runner).to receive(:cache_attributes)
@@ -884,6 +886,49 @@ describe API::Runner do
             expect(response.status).to eq(403)
           end
         end
+
+        context 'when trace is patched' do
+          before do
+            patch_the_trace
+          end
+
+          it 'has valid trace' do
+            expect(response.status).to eq(202)
+            expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
+          end
+
+          context 'when redis data are flushed' do
+            before do
+              redis_shared_state_cleanup!
+            end
+
+            it 'has empty trace' do
+              expect(job.reload.trace.raw).to eq ''
+            end
+
+            context 'when we perform partial patch' do
+              before do
+                patch_the_trace('hello', headers.merge({ 'Content-Range' => "28-32/5" }))
+              end
+
+              it 'returns an error' do
+                expect(response.status).to eq(416)
+                expect(response.header['Range']).to eq('0-0')
+              end
+            end
+
+            context 'when we resend full trace' do
+              before do
+                patch_the_trace('BUILD TRACE appended appended hello', headers.merge({ 'Content-Range' => "0-34/35" }))
+              end
+
+              it 'succeeds with updating trace' do
+                expect(response.status).to eq(202)
+                expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended hello'
+              end
+            end
+          end
+        end
       end
 
       context 'when Runner makes a force-patch' do
@@ -900,7 +945,7 @@ describe API::Runner do
       end
 
       context 'when content-range start is too big' do
-        let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20' }) }
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20/6' }) }
 
         it 'gets 416 error response with range headers' do
           expect(response.status).to eq 416
@@ -910,7 +955,7 @@ describe API::Runner do
       end
 
       context 'when content-range start is too small' do
-        let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20' }) }
+        let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20/13' }) }
 
         it 'gets 416 error response with range headers' do
           expect(response.status).to eq 416

spec/services/ci/retry_build_service_spec.rb

@@ -32,7 +32,7 @@ describe Ci::RetryBuildService do
        runner_id tag_taggings taggings tags trigger_request_id
        user_id auto_canceled_by_id retried failure_reason
        sourced_pipelines artifacts_file_store artifacts_metadata_store
-       metadata].freeze # EE
+       metadata trace_chunks].freeze # EE
 
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }

spec/spec_helper.rb

@@ -93,6 +93,7 @@ RSpec.configure do |config|
   config.include WaitForRequests, :js
   config.include LiveDebugger, :js
   config.include MigrationsHelpers, :migration
+  config.include RedisHelpers
 
   # EE only START
   config.include EE::LicenseHelpers
@@ -165,21 +166,27 @@ RSpec.configure do |config|
   end
 
   config.around(:each, :clean_gitlab_redis_cache) do |example|
-    Gitlab::Redis::Cache.with(&:flushall)
+    redis_cache_cleanup!
 
     example.run
 
-    Gitlab::Redis::Cache.with(&:flushall)
+    redis_cache_cleanup!
   end
 
   config.around(:each, :clean_gitlab_redis_shared_state) do |example|
-    Gitlab::Redis::SharedState.with(&:flushall)
-    Sidekiq.redis(&:flushall)
+    redis_shared_state_cleanup!
 
     example.run
 
-    Gitlab::Redis::SharedState.with(&:flushall)
-    Sidekiq.redis(&:flushall)
+    redis_shared_state_cleanup!
+  end
+
+  config.around(:each, :clean_gitlab_redis_queues) do |example|
+    redis_queues_cleanup!
+
+    example.run
+
+    redis_queues_cleanup!
   end
 
   # The :each scope runs "inside" the example, so this hook ensures the DB is in the

spec/support/chunked_io/chunked_io_helpers.rb

+module ChunkedIOHelpers
+  def sample_trace_raw
+    @sample_trace_raw ||= File.read(expand_fixture_path('trace/sample_trace'))
+      .force_encoding(Encoding::BINARY)
+  end
+
+  def stub_buffer_size(size)
+    stub_const('Ci::BuildTraceChunk::CHUNK_SIZE', size)
+    stub_const('Gitlab::Ci::Trace::ChunkedIO::CHUNK_SIZE', size)
+  end
+end

spec/support/redis/redis_helpers.rb

+module RedisHelpers
+  # config/README.md
+
+  # Usage: performance enhancement
+  def redis_cache_cleanup!
+    Gitlab::Redis::Cache.with(&:flushall)
+  end
+
+  # Usage: SideKiq, Mailroom, CI Runner, Workhorse, push services
+  def redis_queues_cleanup!
+    Gitlab::Redis::Queues.with(&:flushall)
+  end
+
+  # Usage: session state, rate limiting
+  def redis_shared_state_cleanup!
+    Gitlab::Redis::SharedState.with(&:flushall)
+  end
+end

spec/support/shared_examples/ci_trace_shared_examples.rb

+shared_examples_for 'common trace features' do
+  describe '#html' do
+    before do
+      trace.set("12\n34")
+    end
+
+    it "returns formatted html" do
+      expect(trace.html).to eq("12<br>34")
+    end
+
+    it "returns last line of formatted html" do
+      expect(trace.html(last_lines: 1)).to eq("34")
+    end
+  end
+
+  describe '#raw' do
+    before do
+      trace.set("12\n34")
+    end
+
+    it "returns raw output" do
+      expect(trace.raw).to eq("12\n34")
+    end
+
+    it "returns last line of raw output" do
+      expect(trace.raw(last_lines: 1)).to eq("34")
+    end
+  end
+
+  describe '#extract_coverage' do
+    let(:regex) { '\(\d+.\d+\%\) covered' }
+
+    context 'matching coverage' do
+      before do
+        trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
+      end
+
+      it "returns valid coverage" do
+        expect(trace.extract_coverage(regex)).to eq("98.29")
+      end
+    end
+
+    context 'no coverage' do
+      before do
+        trace.set('No coverage')
+      end
+
+      it 'returs nil' do
+        expect(trace.extract_coverage(regex)).to be_nil
+      end
+    end
+  end
+
+  describe '#extract_sections' do
+    let(:log) { 'No sections' }
+    let(:sections) { trace.extract_sections }
+
+    before do
+      trace.set(log)
+    end
+
+    context 'no sections' do
+      it 'returs []' do
+        expect(trace.extract_sections).to eq([])
+      end
+    end
+
+    context 'multiple sections available' do
+      let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) }
+      let(:sections_data) do
+        [
+          { name: 'prepare_script', lines: 2, duration: 3.seconds },
+          { name: 'get_sources', lines: 4, duration: 1.second },
+          { name: 'restore_cache', lines: 0, duration: 0.seconds },
+          { name: 'download_artifacts', lines: 0, duration: 0.seconds },
+          { name: 'build_script', lines: 2, duration: 1.second },
+          { name: 'after_script', lines: 0, duration: 0.seconds },
+          { name: 'archive_cache', lines: 0, duration: 0.seconds },
+          { name: 'upload_artifacts', lines: 0, duration: 0.seconds }
+        ]
+      end
+
+      it "returns valid sections" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(sections_data.size),
+                                 "expected #{sections_data.size} sections, got #{sections.size}"
+
+        buff = StringIO.new(log)
+        sections.each_with_index do |s, i|
+          expected = sections_data[i]
+
+          expect(s[:name]).to eq(expected[:name])
+          expect(s[:date_end] - s[:date_start]).to eq(expected[:duration])
+
+          buff.seek(s[:byte_start], IO::SEEK_SET)
+          length = s[:byte_end] - s[:byte_start]
+          lines = buff.read(length).count("\n")
+          expect(lines).to eq(expected[:lines])
+        end
+      end
+    end
+
+    context 'logs contains "section_start"' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"}
+
+      it "returns only one section" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(1)
+
+        section = sections[0]
+        expect(section[:name]).to eq('a_section')
+        expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section"
+      end
+    end
+
+    context 'missing section_end' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'missing section_start' do
+      let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'inverted section_start section_end' do
+      let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+  end
+
+  describe '#set' do
+    before do
+      trace.set("12")
+    end
+
+    it "returns trace" do
+      expect(trace.raw).to eq("12")
+    end
+
+    context 'overwrite trace' do
+      before do
+        trace.set("34")
+      end
+
+      it "returns new trace" do
+        expect(trace.raw).to eq("34")
+      end
+    end
+
+    context 'runners token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.project.update(runners_token: token)
+        trace.set(token)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+
+    context 'hides build token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.update(token: token)
+        trace.set(token)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+  end
+
+  describe '#append' do
+    before do
+      trace.set("1234")
+    end
+
+    it "returns correct trace" do
+      expect(trace.append("56", 4)).to eq(6)
+      expect(trace.raw).to eq("123456")
+    end
+
+    context 'tries to append trace at different offset' do
+      it "fails with append" do
+        expect(trace.append("56", 2)).to eq(4)
+        expect(trace.raw).to eq("1234")
+      end
+    end
+
+    context 'runners token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.project.update(runners_token: token)
+        trace.append(token, 0)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+
+    context 'build token' do
+      let(:token) { 'my_secret_token' }
+
+      before do
+        build.update(token: token)
+        trace.append(token, 0)
+      end
+
+      it "hides token" do
+        expect(trace.raw).not_to include(token)
+      end
+    end
+  end
+end
+
+shared_examples_for 'trace with disabled live trace feature' do
+  it_behaves_like 'common trace features'
+
+  describe '#read' do
+    shared_examples 'read successfully with IO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(IO)
+        end
+      end
+    end
+
+    shared_examples 'read successfully with StringIO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(StringIO)
+        end
+      end
+    end
+
+    shared_examples 'failed to read' do
+      it 'yields without source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_nil
+        end
+      end
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_id) exists' do
+      before do
+        expect(trace).to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when current_path (with project_ci_id) exists' do
+      before do
+        expect(trace).to receive(:deprecated_path) { expand_fixture_path('trace/sample_trace') }
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when db trace exists' do
+      before do
+        build.send(:write_attribute, :trace, "data")
+      end
+
+      it_behaves_like 'read successfully with StringIO'
+    end
+
+    context 'when no sources exist' do
+      it_behaves_like 'failed to read'
+    end
+  end
+
+  describe 'trace handling' do
+    subject { trace.exist? }
+
+    context 'trace does not exist' do
+      it { expect(trace.exist?).to be(false) }
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it { is_expected.to be_truthy }
+
+      context 'when the trace artifact has been erased' do
+        before do
+          trace.erase!
+        end
+
+        it { is_expected.to be_falsy }
+
+        it 'removes associations' do
+          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
+        end
+      end
+    end
+
+    context 'new trace path is used' do
+      before do
+        trace.send(:ensure_directory)
+
+        File.open(trace.send(:default_path), "w") do |file|
+          file.write("data")
+        end
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+      end
+    end
+
+    context 'deprecated path' do
+      let(:path) { trace.send(:deprecated_path) }
+
+      context 'with valid ci_id' do
+        before do
+          build.project.update(ci_id: 1000)
+
+          FileUtils.mkdir_p(File.dirname(path))
+
+          File.open(path, "w") do |file|
+            file.write("data")
+          end
+        end
+
+        it "trace exist" do
+          expect(trace.exist?).to be(true)
+        end
+
+        it "can be erased" do
+          trace.erase!
+          expect(trace.exist?).to be(false)
+        end
+      end
+
+      context 'without valid ci_id' do
+        it "does not return deprecated path" do
+          expect(path).to be_nil
+        end
+      end
+    end
+
+    context 'stored in database' do
+      before do
+        build.send(:write_attribute, :trace, "data")
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+      end
+
+      it "returns database data" do
+        expect(trace.raw).to eq("data")
+      end
+    end
+  end
+
+  describe '#archive!' do
+    subject { trace.archive! }
+
+    shared_examples 'archive trace file' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(File.exist?(src_path)).to be_falsy
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace file stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        expect(File.exist?(src_path)).to be_truthy
+      end
+    end
+
+    shared_examples 'archive trace in database' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(build.old_trace).to be_nil
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace in database stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        expect(build.old_trace).to eq(trace_content)
+      end
+    end
+
+    context 'when job does not have trace artifact' do
+      context 'when trace file stored in default path' do
+        let!(:build) { create(:ci_build, :success, :trace_live) }
+        let!(:src_path) { trace.read { |s| s.path } }
+        let!(:src_checksum) { Digest::SHA256.file(src_path).hexdigest }
+
+        it_behaves_like 'archive trace file'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace file stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace file stays intact', error: ActiveRecord::RecordInvalid
+        end
+      end
+
+      context 'when trace is stored in database' do
+        let(:build) { create(:ci_build, :success) }
+        let(:trace_content) { 'Sample trace' }
+        let(:src_checksum) { Digest::SHA256.hexdigest(trace_content) }
+
+        before do
+          build.update_column(:trace, trace_content)
+        end
+
+        it_behaves_like 'archive trace in database'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace in database stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace in database stays intact', error: ActiveRecord::RecordInvalid
+        end
+
+        context 'when there is a validation error on Ci::Build' do
+          before do
+            allow_any_instance_of(Ci::Build).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::Build).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          context "when erase old trace with 'save'" do
+            before do
+              build.send(:write_attribute, :trace, nil)
+              build.save
+            end
+
+            it 'old trace is not deleted' do
+              build.reload
+              expect(build.trace.raw).to eq(trace_content)
+            end
+          end
+
+          it_behaves_like 'archive trace in database'
+        end
+      end
+    end
+
+    context 'when job has trace artifact' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Already archived')
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+      end
+    end
+
+    context 'when job is not finished yet' do
+      let!(:build) { create(:ci_build, :running, :trace_live) }
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Job is not finished yet')
+        expect(build.trace.exist?).to be_truthy
+      end
+    end
+  end
+end
+
+shared_examples_for 'trace with enabled live trace feature' do
+  it_behaves_like 'common trace features'
+
+  describe '#read' do
+    shared_examples 'read successfully with IO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(IO)
+        end
+      end
+    end
+
+    shared_examples 'read successfully with ChunkedIO' do
+      it 'yields with source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_a(Gitlab::Ci::Trace::ChunkedIO)
+        end
+      end
+    end
+
+    shared_examples 'failed to read' do
+      it 'yields without source' do
+        trace.read do |stream|
+          expect(stream).to be_a(Gitlab::Ci::Trace::Stream)
+          expect(stream.stream).to be_nil
+        end
+      end
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it_behaves_like 'read successfully with IO'
+    end
+
+    context 'when live trace exists' do
+      before do
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          stream.write('abc')
+        end
+      end
+
+      it_behaves_like 'read successfully with ChunkedIO'
+    end
+
+    context 'when no sources exist' do
+      it_behaves_like 'failed to read'
+    end
+  end
+
+  describe 'trace handling' do
+    subject { trace.exist? }
+
+    context 'trace does not exist' do
+      it { expect(trace.exist?).to be(false) }
+    end
+
+    context 'when trace artifact exists' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it { is_expected.to be_truthy }
+
+      context 'when the trace artifact has been erased' do
+        before do
+          trace.erase!
+        end
+
+        it { is_expected.to be_falsy }
+
+        it 'removes associations' do
+          expect(Ci::JobArtifact.exists?(job_id: build.id, file_type: :trace)).to be_falsy
+        end
+      end
+    end
+
+    context 'stored in live trace' do
+      before do
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          stream.write('abc')
+        end
+      end
+
+      it "trace exist" do
+        expect(trace.exist?).to be(true)
+      end
+
+      it "can be erased" do
+        trace.erase!
+        expect(trace.exist?).to be(false)
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
+      end
+
+      it "returns live trace data" do
+        expect(trace.raw).to eq("abc")
+      end
+    end
+  end
+
+  describe '#archive!' do
+    subject { trace.archive! }
+
+    shared_examples 'archive trace file in ChunkedIO' do
+      it do
+        expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+        expect(build.job_artifacts_trace.file.filename).to eq('job.log')
+        expect(Ci::BuildTraceChunk.where(build: build)).not_to be_exist
+        expect(src_checksum)
+          .to eq(Digest::SHA256.file(build.job_artifacts_trace.file.path).hexdigest)
+        expect(build.job_artifacts_trace.file_sha256).to eq(src_checksum)
+      end
+    end
+
+    shared_examples 'source trace in ChunkedIO stays intact' do |error:|
+      it do
+        expect { subject }.to raise_error(error)
+
+        build.reload
+        expect(build.trace.exist?).to be_truthy
+        expect(build.job_artifacts_trace).to be_nil
+        Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream|
+          expect(stream.read).to eq(trace_raw)
+        end
+      end
+    end
+
+    context 'when job does not have trace artifact' do
+      context 'when trace is stored in ChunkedIO' do
+        let!(:build) { create(:ci_build, :success, :trace_live) }
+        let!(:trace_raw) { build.trace.raw }
+        let!(:src_checksum) { Digest::SHA256.hexdigest(trace_raw) }
+
+        it_behaves_like 'archive trace file in ChunkedIO'
+
+        context 'when failed to create clone file' do
+          before do
+            allow(IO).to receive(:copy_stream).and_return(0)
+          end
+
+          it_behaves_like 'source trace in ChunkedIO stays intact', error: Gitlab::Ci::Trace::ArchiveError
+        end
+
+        context 'when failed to create job artifact record' do
+          before do
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return(%w[Error Error])
+          end
+
+          it_behaves_like 'source trace in ChunkedIO stays intact', error: ActiveRecord::RecordInvalid
+        end
+      end
+    end
+
+    context 'when job has trace artifact' do
+      before do
+        create(:ci_job_artifact, :trace, job: build)
+      end
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Already archived')
+        expect(build.job_artifacts_trace.file.exists?).to be_truthy
+      end
+    end
+
+    context 'when job is not finished yet' do
+      let!(:build) { create(:ci_build, :running, :trace_live) }
+
+      it 'does not archive' do
+        expect_any_instance_of(described_class).not_to receive(:archive_stream!)
+        expect { subject }.to raise_error('Job is not finished yet')
+        expect(build.trace.exist?).to be_truthy
+      end
+    end
+  end
+end

spec/support/shared_examples/fast_destroy_all.rb

+shared_examples_for 'fast destroyable' do
+  describe 'Forbid #destroy and #destroy_all' do
+    it 'does not delete database rows and associted external data' do
+      expect(external_data_counter).to be > 0
+      expect(subjects.count).to be > 0
+
+      expect { subjects.first.destroy }.to raise_error('`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`')
+      expect { subjects.destroy_all }.to raise_error('`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`')
+
+      expect(subjects.count).to be > 0
+      expect(external_data_counter).to be > 0
+    end
+  end
+
+  describe '.fast_destroy_all' do
+    it 'deletes database rows and associted external data' do
+      expect(external_data_counter).to be > 0
+      expect(subjects.count).to be > 0
+
+      expect { subjects.fast_destroy_all }.not_to raise_error
+
+      expect(subjects.count).to eq(0)
+      expect(external_data_counter).to eq(0)
+    end
+  end
+
+  describe '.use_fast_destroy' do
+    it 'performs cascading delete with fast_destroy_all' do
+      expect(external_data_counter).to be > 0
+      expect(subjects.count).to be > 0
+
+      expect { parent.destroy }.not_to raise_error
+
+      expect(subjects.count).to eq(0)
+      expect(external_data_counter).to eq(0)
+    end
+  end
+end