Merge branch 'add-ansi2json-log-parser' into 'master'

Introduce new Ansi2json parser

See merge request gitlab-org/gitlab!18133

changelogs/unreleased/add-ansi2json-log-parser.yml

+---
+title: Introduce new Ansi2json parser to convert job logs to JSON
+merge_request: 18133
+author:
+type: added

lib/gitlab/ci/ansi2json.rb

+# frozen_string_literal: true
+
+# Convert terminal stream to JSON
+module Gitlab
+  module Ci
+    module Ansi2json
+      def self.convert(ansi, state = nil)
+        Converter.new.convert(ansi, state)
+      end
+    end
+  end
+end

lib/gitlab/ci/ansi2json/converter.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Converter
+        def convert(stream, new_state)
+          @lines = []
+          @state = State.new(new_state, stream.size)
+
+          append = false
+          truncated = false
+
+          cur_offset = stream.tell
+          if cur_offset > @state.offset
+            @state.offset = cur_offset
+            truncated = true
+          else
+            stream.seek(@state.offset)
+            append = @state.offset > 0
+          end
+
+          start_offset = @state.offset
+
+          @state.set_current_line!(style: Style.new(@state.inherited_style))
+
+          stream.each_line do |line|
+            s = StringScanner.new(line)
+            convert_line(s)
+          end
+
+          # This must be assigned before flushing the current line
+          # or the @current_line.offset will advance to the very end
+          # of the trace. Instead we want @last_line_offset to always
+          # point to the beginning of last line.
+          @state.set_last_line_offset
+
+          flush_current_line
+
+          OpenStruct.new(
+            lines: @lines,
+            state: @state.encode,
+            append: append,
+            truncated: truncated,
+            offset: start_offset,
+            size: stream.tell - start_offset,
+            total: stream.size
+          )
+        end
+
+        private
+
+        def convert_line(scanner)
+          until scanner.eos?
+
+            if scanner.scan(Gitlab::Regex.build_trace_section_regex)
+              handle_section(scanner)
+            elsif scanner.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(scanner)
+            elsif scanner.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif scanner.scan(/</)
+              @state.current_line << '&lt;'
+            elsif scanner.scan(/\r?\n/)
+              # we advance the offset of the next current line
+              # so it does not start from \n
+              flush_current_line(advance_offset: scanner.matched_size)
+            else
+              @state.current_line << scanner.scan(/./m)
+            end
+
+            @state.offset += scanner.matched_size
+          end
+        end
+
+        def handle_sequence(scanner)
+          indicator = scanner[1]
+          commands = scanner[2].split ';'
+          terminator = scanner[3]
+
+          # We are only interested in color and text style changes - triggered by
+          # sequences starting with '\e[' and ending with 'm'. Any other control
+          # sequence gets stripped (including stuff like "delete last line")
+          return unless indicator == '[' && terminator == 'm'
+
+          @state.update_style(commands)
+        end
+
+        def handle_section(scanner)
+          action = scanner[1]
+          timestamp = scanner[2]
+          section = scanner[3]
+
+          section_name = sanitize_section_name(section)
+
+          if action == "start"
+            handle_section_start(section_name, timestamp)
+          elsif action == "end"
+            handle_section_end(section_name, timestamp)
+          end
+        end
+
+        def handle_section_start(section, timestamp)
+          flush_current_line unless @state.current_line.empty?
+          @state.open_section(section, timestamp)
+        end
+
+        def handle_section_end(section, timestamp)
+          return unless @state.section_open?(section)
+
+          flush_current_line unless @state.current_line.empty?
+          @state.close_section(section, timestamp)
+
+          # ensure that section end is detached from the last
+          # line in the section
+          flush_current_line
+        end
+
+        def flush_current_line(advance_offset: 0)
+          @lines << @state.current_line.to_h
+
+          @state.set_current_line!(advance_offset: advance_offset)
+        end
+
+        def sanitize_section_name(section)
+          section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/ansi2json/line.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      # Line class is responsible for keeping the internal state of
+      # a log line and to finally serialize it as Hash.
+      class Line
+        # Line::Segment is a portion of a line that has its own style
+        # and text. Multiple segments make the line content.
+        class Segment
+          attr_accessor :text, :style
+
+          def initialize(style:)
+            @text = +''
+            @style = style
+          end
+
+          def empty?
+            text.empty?
+          end
+
+          def to_h
+            # Without force encoding to UTF-8 we could get an error
+            # when serializing the Hash to JSON.
+            # Encoding::UndefinedConversionError:
+            #   "\xE2" from ASCII-8BIT to UTF-8
+            { text: text.force_encoding('UTF-8') }.tap do |result|
+              result[:style] = style.to_s if style.set?
+            end
+          end
+        end
+
+        attr_reader :offset, :sections, :segments, :current_segment,
+                    :section_header, :section_duration
+
+        def initialize(offset:, style:, sections: [])
+          @offset = offset
+          @segments = []
+          @sections = sections
+          @section_header = false
+          @duration = nil
+          @current_segment = Segment.new(style: style)
+        end
+
+        def <<(data)
+          @current_segment.text << data
+        end
+
+        def style
+          @current_segment.style
+        end
+
+        def empty?
+          @segments.empty? && @current_segment.empty?
+        end
+
+        def update_style(ansi_commands)
+          @current_segment.style.update(ansi_commands)
+        end
+
+        def add_section(section)
+          @sections << section
+        end
+
+        def set_as_section_header
+          @section_header = true
+        end
+
+        def set_section_duration(duration)
+          @section_duration = Time.at(duration.to_i).strftime('%M:%S')
+        end
+
+        def flush_current_segment!
+          return if @current_segment.empty?
+
+          @segments << @current_segment.to_h
+          @current_segment = Segment.new(style: @current_segment.style)
+        end
+
+        def to_h
+          flush_current_segment!
+
+          { offset: offset, content: @segments }.tap do |result|
+            result[:section] = sections.last if sections.any?
+            result[:section_header] = true if @section_header
+            result[:section_duration] = @section_duration if @section_duration
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/ansi2json/parser.rb

+# frozen_string_literal: true
+
+# This Parser translates ANSI escape codes into human readable format.
+# It considers color and format changes.
+# Inspired by http://en.wikipedia.org/wiki/ANSI_escape_code
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Parser
+        # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+        COLOR = {
+          0 => 'black', # not that this is gray in the intense color table
+          1 => 'red',
+          2 => 'green',
+          3 => 'yellow',
+          4 => 'blue',
+          5 => 'magenta',
+          6 => 'cyan',
+          7 => 'white' # not that this is gray in the dark (aka default) color table
+        }.freeze
+
+        STYLE_SWITCHES = {
+          bold:       0x01,
+          italic:     0x02,
+          underline:  0x04,
+          conceal:    0x08,
+          cross:      0x10
+        }.freeze
+
+        def self.bold?(mask)
+          mask & STYLE_SWITCHES[:bold] != 0
+        end
+
+        def self.matching_formats(mask)
+          formats = []
+          STYLE_SWITCHES.each do |text_format, flag|
+            formats << "term-#{text_format}" if mask & flag != 0
+          end
+
+          formats
+        end
+
+        def initialize(command, ansi_stack = nil)
+          @command = command
+          @ansi_stack = ansi_stack
+        end
+
+        def changes
+          if self.respond_to?("on_#{@command}")
+            send("on_#{@command}", @ansi_stack) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        # rubocop:disable Style/SingleLineMethods
+        def on_0(_) { reset: true } end
+
+        def on_1(_) { enable: STYLE_SWITCHES[:bold] } end
+
+        def on_3(_) { enable: STYLE_SWITCHES[:italic] } end
+
+        def on_4(_) { enable: STYLE_SWITCHES[:underline] } end
+
+        def on_8(_) { enable: STYLE_SWITCHES[:conceal] } end
+
+        def on_9(_) { enable: STYLE_SWITCHES[:cross] } end
+
+        def on_21(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_22(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_23(_) { disable: STYLE_SWITCHES[:italic] } end
+
+        def on_24(_) { disable: STYLE_SWITCHES[:underline] } end
+
+        def on_28(_) { disable: STYLE_SWITCHES[:conceal] } end
+
+        def on_29(_) { disable: STYLE_SWITCHES[:cross] } end
+
+        def on_30(_) { fg: fg_color(0) } end
+
+        def on_31(_) { fg: fg_color(1) } end
+
+        def on_32(_) { fg: fg_color(2) } end
+
+        def on_33(_) { fg: fg_color(3) } end
+
+        def on_34(_) { fg: fg_color(4) } end
+
+        def on_35(_) { fg: fg_color(5) } end
+
+        def on_36(_) { fg: fg_color(6) } end
+
+        def on_37(_) { fg: fg_color(7) } end
+
+        def on_38(stack) { fg: fg_color_256(stack) } end
+
+        def on_39(_) { fg: fg_color(9) } end
+
+        def on_40(_) { bg: bg_color(0) } end
+
+        def on_41(_) { bg: bg_color(1) } end
+
+        def on_42(_) { bg: bg_color(2) } end
+
+        def on_43(_) { bg: bg_color(3) } end
+
+        def on_44(_) { bg: bg_color(4) } end
+
+        def on_45(_) { bg: bg_color(5) } end
+
+        def on_46(_) { bg: bg_color(6) } end
+
+        def on_47(_) { bg: bg_color(7) } end
+
+        def on_48(stack) { bg: bg_color_256(stack) } end
+
+        # TODO: all the x9 never get called?
+        def on_49(_) { fg: fg_color(9) } end
+
+        def on_90(_) { fg: fg_color(0, 'l') } end
+
+        def on_91(_) { fg: fg_color(1, 'l') } end
+
+        def on_92(_) { fg: fg_color(2, 'l') } end
+
+        def on_93(_) { fg: fg_color(3, 'l') } end
+
+        def on_94(_) { fg: fg_color(4, 'l') } end
+
+        def on_95(_) { fg: fg_color(5, 'l') } end
+
+        def on_96(_) { fg: fg_color(6, 'l') } end
+
+        def on_97(_) { fg: fg_color(7, 'l') } end
+
+        def on_99(_) { fg: fg_color(9, 'l') } end
+
+        def on_100(_) { fg: bg_color(0, 'l') } end
+
+        def on_101(_) { fg: bg_color(1, 'l') } end
+
+        def on_102(_) { fg: bg_color(2, 'l') } end
+
+        def on_103(_) { fg: bg_color(3, 'l') } end
+
+        def on_104(_) { fg: bg_color(4, 'l') } end
+
+        def on_105(_) { fg: bg_color(5, 'l') } end
+
+        def on_106(_) { fg: bg_color(6, 'l') } end
+
+        def on_107(_) { fg: bg_color(7, 'l') } end
+
+        def on_109(_) { fg: bg_color(9, 'l') } end
+        # rubocop:enable Style/SingleLineMethods
+
+        def fg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['fg', prefix])
+        end
+
+        def fg_color_256(command_stack)
+          xterm_color_class(command_stack, 'fg')
+        end
+
+        def bg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['bg', prefix])
+        end
+
+        def bg_color_256(command_stack)
+          xterm_color_class(command_stack, 'bg')
+        end
+
+        def term_color_class(color_index, prefix)
+          color_name = COLOR[color_index]
+          return if color_name.nil?
+
+          color_class(['term', prefix, color_name])
+        end
+
+        def xterm_color_class(command_stack, prefix)
+          # the 38 and 48 commands have to be followed by "5" and the color index
+          return unless command_stack.length >= 2
+          return unless command_stack[0] == "5"
+
+          command_stack.shift # ignore the "5" command
+          color_index = command_stack.shift.to_i
+
+          return unless color_index >= 0
+          return unless color_index <= 255
+
+          color_class(["xterm", prefix, color_index])
+        end
+
+        def color_class(segments)
+          [segments].flatten.compact.join('-')
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/ansi2json/state.rb

+# frozen_string_literal: true
+
+# In this class we keep track of the state changes that the
+# Converter makes as it scans through the log stream.
+module Gitlab
+  module Ci
+    module Ansi2json
+      class State
+        attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset
+
+        def initialize(new_state, stream_size)
+          @offset = 0
+          @inherited_style = {}
+          @open_sections = {}
+          @stream_size = stream_size
+
+          restore_state!(new_state)
+        end
+
+        def encode
+          state = {
+            offset: @last_line_offset,
+            style: @current_line.style.to_h,
+            open_sections: @open_sections
+          }
+          Base64.urlsafe_encode64(state.to_json)
+        end
+
+        def open_section(section, timestamp)
+          @open_sections[section] = timestamp
+
+          @current_line.add_section(section)
+          @current_line.set_as_section_header
+        end
+
+        def close_section(section, timestamp)
+          return unless section_open?(section)
+
+          duration = timestamp.to_i - @open_sections[section].to_i
+          @current_line.set_section_duration(duration)
+
+          @open_sections.delete(section)
+        end
+
+        def section_open?(section)
+          @open_sections.key?(section)
+        end
+
+        def set_current_line!(style: nil, advance_offset: 0)
+          new_line = Line.new(
+            offset: @offset + advance_offset,
+            style: style || @current_line.style,
+            sections: @open_sections.keys
+          )
+          @current_line = new_line
+        end
+
+        def set_last_line_offset
+          @last_line_offset = @current_line.offset
+        end
+
+        def update_style(commands)
+          @current_line.flush_current_segment!
+          @current_line.update_style(commands)
+        end
+
+        private
+
+        def restore_state!(encoded_state)
+          state = decode_state(encoded_state)
+
+          return unless state
+          return if state['offset'].to_i > @stream_size
+
+          @offset = state['offset'].to_i if state['offset']
+          @open_sections = state['open_sections'] if state['open_sections']
+
+          if state['style']
+            @inherited_style = {
+              fg: state.dig('style', 'fg'),
+              bg: state.dig('style', 'bg'),
+              mask: state.dig('style', 'mask')
+            }
+          end
+        end
+
+        def decode_state(state)
+          return unless state.present?
+
+          decoded_state = Base64.urlsafe_decode64(state)
+          return unless decoded_state.present?
+
+          JSON.parse(decoded_state)
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/ansi2json/style.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Style
+        attr_reader :fg, :bg, :mask
+
+        def initialize(fg: nil, bg: nil, mask: 0)
+          @fg = fg
+          @bg = bg
+          @mask = mask
+
+          update_formats
+        end
+
+        def update(ansi_commands)
+          command = ansi_commands.shift
+          return unless command
+
+          if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes
+            apply_changes(changes)
+          end
+
+          update(ansi_commands)
+        end
+
+        def set?
+          @fg || @bg || @formats.any?
+        end
+
+        def reset!
+          @fg = nil
+          @bg = nil
+          @mask = 0
+          @formats = []
+        end
+
+        def ==(other)
+          self.to_h == other.to_h
+        end
+
+        def to_s
+          [@fg, @bg, @formats].flatten.compact.join(' ')
+        end
+
+        def to_h
+          { fg: @fg, bg: @bg, mask: @mask }
+        end
+
+        private
+
+        def apply_changes(changes)
+          case
+          when changes[:reset]
+            reset!
+          when changes[:fg]
+            @fg = changes[:fg]
+          when changes[:bg]
+            @bg = changes[:bg]
+          when changes[:enable]
+            @mask |= changes[:enable]
+          when changes[:disable]
+            @mask &= ~changes[:disable]
+          else
+            return
+          end
+
+          update_formats
+        end
+
+        def update_formats
+          # Most terminals show bold colored text in the light color variant
+          # Let's mimic that here
+          if @fg.present? && Gitlab::Ci::Ansi2json::Parser.bold?(@mask)
+            @fg = @fg.sub(/fg-([a-z]{2,}+)/, 'fg-l-\1')
+          end
+
+          @formats = Gitlab::Ci::Ansi2json::Parser.matching_formats(@mask)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/ansi2json/line_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Ansi2json::Line do
+  let(:offset) { 0 }
+  let(:style) { Gitlab::Ci::Ansi2json::Style.new }
+
+  subject { described_class.new(offset: offset, style: style) }
+
+  describe '#<<' do
+    it 'appends new data to the current segment' do
+      expect { subject << 'test 1' }.to change { subject.current_segment.text }
+      expect(subject.current_segment.text).to eq('test 1')
+
+      expect { subject << ', test 2' }.to change { subject.current_segment.text }
+      expect(subject.current_segment.text).to eq('test 1, test 2')
+    end
+  end
+
+  describe '#style' do
+    context 'when style is passed to the initializer' do
+      let(:style) { double }
+
+      it 'returns the same style' do
+        expect(subject.style).to eq(style)
+      end
+    end
+
+    context 'when style is not passed to the initializer' do
+      it 'returns the default style' do
+        expect(subject.style.set?).to be_falsey
+      end
+    end
+  end
+
+  describe '#update_style' do
+    let(:expected_style) do
+      Gitlab::Ci::Ansi2json::Style.new(
+        fg: 'term-fg-l-yellow',
+        bg: 'term-bg-blue',
+        mask: 1)
+    end
+
+    it 'sets the style' do
+      subject.update_style(%w[1 33 44])
+
+      expect(subject.style).to eq(expected_style)
+    end
+  end
+
+  describe '#add_section' do
+    it 'appends a new section to the list' do
+      subject.add_section('section_1')
+      subject.add_section('section_2')
+
+      expect(subject.sections).to eq(%w[section_1 section_2])
+    end
+  end
+
+  describe '#set_as_section_header' do
+    it 'change the section_header to true' do
+      expect { subject.set_as_section_header }
+        .to change { subject.section_header }
+        .to be_truthy
+    end
+  end
+
+  describe '#set_section_duration' do
+    it 'sets and formats the section_duration' do
+      subject.set_section_duration(75)
+
+      expect(subject.section_duration).to eq('01:15')
+    end
+  end
+
+  describe '#flush_current_segment!' do
+    context 'when current segment is not empty' do
+      before do
+        subject << 'some data'
+      end
+
+      it 'adds the segment to the list' do
+        expect { subject.flush_current_segment! }.to change { subject.segments.count }.by(1)
+
+        expect(subject.segments.map { |s| s[:text] }).to eq(['some data'])
+      end
+
+      it 'updates the current segment pointer propagating the style' do
+        previous_segment = subject.current_segment
+
+        subject.flush_current_segment!
+
+        expect(subject.current_segment).not_to eq(previous_segment)
+        expect(subject.current_segment.style).to eq(previous_segment.style)
+      end
+    end
+
+    context 'when current segment is empty' do
+      it 'does not add any segments to the list' do
+        expect { subject.flush_current_segment! }.not_to change { subject.segments.count }
+      end
+
+      it 'does not change the current segment' do
+        expect { subject.flush_current_segment! }.not_to change { subject.current_segment }
+      end
+    end
+  end
+
+  describe '#to_h' do
+    before do
+      subject << 'some data'
+      subject.update_style(['1'])
+    end
+
+    context 'when sections are present' do
+      before do
+        subject.add_section('section_1')
+        subject.add_section('section_2')
+      end
+
+      context 'when section header is set' do
+        before do
+          subject.set_as_section_header
+        end
+
+        it 'serializes the attributes set' do
+          result = {
+            offset: 0,
+            content: [{ text: 'some data', style: 'term-bold' }],
+            section: 'section_2',
+            section_header: true
+          }
+
+          expect(subject.to_h).to eq(result)
+        end
+      end
+
+      context 'when section duration is set' do
+        before do
+          subject.set_section_duration(75)
+        end
+
+        it 'serializes the attributes set' do
+          result = {
+            offset: 0,
+            content: [{ text: 'some data', style: 'term-bold' }],
+            section: 'section_2',
+            section_duration: '01:15'
+          }
+
+          expect(subject.to_h).to eq(result)
+        end
+      end
+    end
+
+    context 'when there are no sections' do
+      it 'serializes the attributes set' do
+        result = {
+          offset: 0,
+          content: [{ text: 'some data', style: 'term-bold' }]
+        }
+
+        expect(subject.to_h).to eq(result)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/ansi2json/parser_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+# The rest of the specs for this class are covered in style_spec.rb
+describe Gitlab::Ci::Ansi2json::Parser do
+  subject { described_class }
+
+  describe 'bold?' do
+    it 'returns true if style mask matches bold format' do
+      expect(subject.bold?(0x01)).to be_truthy
+    end
+
+    it 'returns false if style mask does not match bold format' do
+      expect(subject.bold?(0x02)).to be_falsey
+    end
+  end
+
+  describe 'matching_formats' do
+    it 'returns matching formats given a style mask' do
+      expect(subject.matching_formats(0x01)).to eq(%w[term-bold])
+      expect(subject.matching_formats(0x03)).to eq(%w[term-bold term-italic])
+      expect(subject.matching_formats(0x07)).to eq(%w[term-bold term-italic term-underline])
+    end
+
+    it 'returns an empty array if no formats match the style mask' do
+      expect(subject.matching_formats(0)).to eq([])
+    end
+  end
+end

spec/lib/gitlab/ci/ansi2json/style_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Ansi2json::Style do
+  describe '#set?' do
+    subject { described_class.new(params).set? }
+
+    context 'when fg color is set' do
+      let(:params) { { fg: 'term-fg-black' } }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when bg color is set' do
+      let(:params) { { bg: 'term-bg-black' } }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when mask is set' do
+      let(:params) { { mask: 0x01 } }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'nothing is set' do
+      let(:params) { {} }
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#reset!' do
+    let(:style) { described_class.new(fg: 'term-fg-black', bg: 'term-bg-yellow', mask: 0x01) }
+
+    it 'set the style params to default' do
+      style.reset!
+
+      expect(style.fg).to be_nil
+      expect(style.bg).to be_nil
+      expect(style.mask).to be_zero
+    end
+  end
+
+  describe 'update formats to mimic terminals' do
+    subject { described_class.new(params) }
+
+    context 'when fg color present' do
+      let(:params) { { fg: 'term-fg-black', mask: mask } }
+
+      context 'when mask is set to bold' do
+        let(:mask) { 0x01 }
+
+        it 'changes the fg color to a lighter version' do
+          expect(subject.fg).to eq('term-fg-l-black')
+        end
+      end
+
+      context 'when mask set to another format' do
+        let(:mask) { 0x02 }
+
+        it 'does not change the fg color' do
+          expect(subject.fg).to eq('term-fg-black')
+        end
+      end
+
+      context 'when mask is not set' do
+        let(:mask) { 0 }
+
+        it 'does not change the fg color' do
+          expect(subject.fg).to eq('term-fg-black')
+        end
+      end
+    end
+  end
+
+  describe '#update' do
+    where(:initial_state, :ansi_commands, :result, :description) do
+      [
+        # add format
+        [[], %w[0], '', 'does not set any style'],
+        [[], %w[1], 'term-bold', 'enables format bold'],
+        [[], %w[3], 'term-italic', 'enables format italic'],
+        [[], %w[4], 'term-underline', 'enables format underline'],
+        [[], %w[8], 'term-conceal', 'enables format conceal'],
+        [[], %w[9], 'term-cross', 'enables format cross'],
+        # remove format
+        [%w[1], %w[21], '', 'disables format bold'],
+        [%w[1 3], %w[21], 'term-italic', 'disables format bold and leaves italic'],
+        [%w[1], %w[22], '', 'disables format bold using command 22'],
+        [%w[1 3], %w[22], 'term-italic', 'disables format bold and leaves italic using command 22'],
+        [%w[3], %w[23], '', 'disables format italic'],
+        [%w[1 3], %w[23], 'term-bold', 'disables format italic and leaves bold'],
+        [%w[4], %w[24], '', 'disables format underline'],
+        [%w[1 4], %w[24], 'term-bold', 'disables format underline and leaves bold'],
+        [%w[8], %w[28], '', 'disables format conceal'],
+        [%w[1 8], %w[28], 'term-bold', 'disables format conceal and leaves bold'],
+        [%w[9], %w[29], '', 'disables format cross'],
+        [%w[1 9], %w[29], 'term-bold', 'disables format cross and leaves bold'],
+        # set fg color
+        [[], %w[30], 'term-fg-black', 'sets fg color black'],
+        [[], %w[31], 'term-fg-red', 'sets fg color red'],
+        [[], %w[32], 'term-fg-green', 'sets fg color green'],
+        [[], %w[33], 'term-fg-yellow', 'sets fg color yellow'],
+        [[], %w[34], 'term-fg-blue', 'sets fg color blue'],
+        [[], %w[35], 'term-fg-magenta', 'sets fg color magenta'],
+        [[], %w[36], 'term-fg-cyan', 'sets fg color cyan'],
+        [[], %w[37], 'term-fg-white', 'sets fg color white'],
+        # sets xterm fg color
+        [[], %w[38 5 1], 'xterm-fg-1', 'sets xterm fg color 1'],
+        [[], %w[38 5 2], 'xterm-fg-2', 'sets xterm fg color 2'],
+        [[], %w[38 1], 'term-bold', 'ignores 38 command if not followed by 5 and sets format bold'],
+        # set bg color
+        [[], %w[40], 'term-bg-black', 'sets bg color black'],
+        [[], %w[41], 'term-bg-red', 'sets bg color red'],
+        [[], %w[42], 'term-bg-green', 'sets bg color green'],
+        [[], %w[43], 'term-bg-yellow', 'sets bg color yellow'],
+        [[], %w[44], 'term-bg-blue', 'sets bg color blue'],
+        [[], %w[45], 'term-bg-magenta', 'sets bg color magenta'],
+        [[], %w[46], 'term-bg-cyan', 'sets bg color cyan'],
+        [[], %w[47], 'term-bg-white', 'sets bg color white'],
+        # set xterm bg color
+        [[], %w[48 5 1], 'xterm-bg-1', 'sets xterm bg color 1'],
+        [[], %w[48 5 2], 'xterm-bg-2', 'sets xterm bg color 2'],
+        [[], %w[48 1], 'term-bold', 'ignores 48 command if not followed by 5 and sets format bold'],
+        # set light fg color
+        [[], %w[90], 'term-fg-l-black', 'sets fg color light black'],
+        [[], %w[91], 'term-fg-l-red', 'sets fg color light red'],
+        [[], %w[92], 'term-fg-l-green', 'sets fg color light green'],
+        [[], %w[93], 'term-fg-l-yellow', 'sets fg color light yellow'],
+        [[], %w[94], 'term-fg-l-blue', 'sets fg color light blue'],
+        [[], %w[95], 'term-fg-l-magenta', 'sets fg color light magenta'],
+        [[], %w[96], 'term-fg-l-cyan', 'sets fg color light cyan'],
+        [[], %w[97], 'term-fg-l-white', 'sets fg color light white'],
+        # set light bg color
+        [[], %w[100], 'term-bg-l-black', 'sets bg color light black'],
+        [[], %w[101], 'term-bg-l-red', 'sets bg color light red'],
+        [[], %w[102], 'term-bg-l-green', 'sets bg color light green'],
+        [[], %w[103], 'term-bg-l-yellow', 'sets bg color light yellow'],
+        [[], %w[104], 'term-bg-l-blue', 'sets bg color light blue'],
+        [[], %w[105], 'term-bg-l-magenta', 'sets bg color light magenta'],
+        [[], %w[106], 'term-bg-l-cyan', 'sets bg color light cyan'],
+        [[], %w[107], 'term-bg-l-white', 'sets bg color light white'],
+        # reset
+        [%w[1], %w[0], '', 'resets style from format bold'],
+        [%w[1 3], %w[0], '', 'resets style from format bold and italic'],
+        [%w[1 3 term-fg-l-red term-bg-yellow], %w[0], '', 'resets all formats and colors'],
+        # misc
+        [[], %w[1 30 42 3], 'term-fg-l-black term-bg-green term-bold term-italic', 'adds fg color, bg color and formats from no style'],
+        [%w[3 31], %w[23 1 43], 'term-fg-l-red term-bg-yellow term-bold', 'replaces format italic with bold and adds a yellow background']
+      ]
+    end
+
+    with_them do
+      it 'change the style' do
+        style = described_class.new
+        style.update(initial_state)
+
+        style.update(ansi_commands)
+
+        expect(style.to_s).to eq(result)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/ansi2json_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Ansi2json do
+  subject { described_class }
+
+  describe 'lines' do
+    it 'prints non-ansi as-is' do
+      expect(convert_json('Hello')).to eq([
+        { offset: 0, content: [{ text: 'Hello' }] }
+      ])
+    end
+
+    it 'adds new line in a separate element' do
+      expect(convert_json("Hello\nworld")).to eq([
+        { offset: 0, content: [{ text: 'Hello' }] },
+        { offset: 6, content: [{ text: 'world' }] }
+      ])
+    end
+
+    it 'recognizes color changing ANSI sequences' do
+      expect(convert_json("\e[31mHello\e[0m")).to eq([
+        { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] }
+      ])
+    end
+
+    it 'recognizes color changing ANSI sequences across multiple lines' do
+      expect(convert_json("\e[31mHello\nWorld\e[0m")).to eq([
+        { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red' }] },
+        { offset: 11, content: [{ text: 'World', style: 'term-fg-red' }] }
+      ])
+    end
+
+    it 'recognizes background and foreground colors' do
+      expect(convert_json("\e[31;44mHello")).to eq([
+        { offset: 0, content: [{ text: 'Hello', style: 'term-fg-red term-bg-blue' }] }
+      ])
+    end
+
+    it 'recognizes style changes within the same line' do
+      expect(convert_json("\e[31;44mHello\e[0m world")).to eq([
+        { offset: 0, content: [
+          { text: 'Hello', style: 'term-fg-red term-bg-blue' },
+          { text: ' world' }
+        ] }
+      ])
+    end
+
+    context 'with section markers' do
+      let(:section_name) { 'prepare-script' }
+      let(:section_duration) { 63.seconds }
+      let(:section_start_time) { Time.new(2019, 9, 17).utc }
+      let(:section_end_time) { section_start_time + section_duration }
+      let(:section_start) { "section_start:#{section_start_time.to_i}:#{section_name}\r\033[0K"}
+      let(:section_end) { "section_end:#{section_end_time.to_i}:#{section_name}\r\033[0K"}
+
+      it 'marks the first line of the section as header' do
+        expect(convert_json("Hello#{section_start}world!")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'Hello' }]
+          },
+          {
+            offset: 5,
+            content: [{ text: 'world!' }],
+            section: 'prepare-script',
+            section_header: true
+          }
+        ])
+      end
+
+      it 'does not marks the other lines of the section as header' do
+        expect(convert_json("outside section#{section_start}Hello\nworld!")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'outside section' }]
+          },
+          {
+            offset: 15,
+            content: [{ text: 'Hello' }],
+            section: 'prepare-script',
+            section_header: true
+          },
+          {
+            offset: 65,
+            content: [{ text: 'world!' }],
+            section: 'prepare-script'
+          }
+        ])
+      end
+
+      it 'marks the last line of the section as footer' do
+        expect(convert_json("#{section_start}Good\nmorning\nworld!#{section_end}")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'Good' }],
+            section: 'prepare-script',
+            section_header: true
+          },
+          {
+            offset: 49,
+            content: [{ text: 'morning' }],
+            section: 'prepare-script'
+          },
+          {
+            offset: 57,
+            content: [{ text: 'world!' }],
+            section: 'prepare-script'
+          },
+          {
+            offset: 63,
+            content: [],
+            section_duration: '01:03',
+            section: 'prepare-script'
+          },
+          {
+            offset: 63,
+            content: []
+          }
+        ])
+      end
+
+      it 'marks the first line as header and footer if is the only line in the section' do
+        expect(convert_json("#{section_start}Hello world!#{section_end}")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'Hello world!' }],
+            section: 'prepare-script',
+            section_header: true
+          },
+          {
+            offset: 56,
+            content: [],
+            section: 'prepare-script',
+            section_duration: '01:03'
+          },
+          {
+            offset: 56,
+            content: []
+          }
+        ])
+      end
+
+      it 'does not add sections attribute to lines after the section is closed' do
+        expect(convert_json("#{section_start}Hello#{section_end}world")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'Hello' }],
+            section: 'prepare-script',
+            section_header: true
+          },
+          {
+            offset: 49,
+            content: [],
+            section: 'prepare-script',
+            section_duration: '01:03'
+          },
+          {
+            offset: 49,
+            content: [{ text: 'world' }]
+          }
+        ])
+      end
+
+      it 'ignores section_end marker if no section_start exists' do
+        expect(convert_json("Hello #{section_end}world")).to eq([
+          {
+            offset: 0,
+            content: [{ text: 'Hello world' }]
+          }
+        ])
+      end
+
+      context 'when section name contains .-_ and capital letters' do
+        let(:section_name) { 'a.Legit-SeCtIoN_namE' }
+
+        it 'sanitizes the section name' do
+          expect(convert_json("Hello#{section_start}world!")).to eq([
+            {
+              offset: 0,
+              content: [{ text: 'Hello' }]
+            },
+            {
+              offset: 5,
+              content: [{ text: 'world!' }],
+              section: 'a-legit-section-name',
+              section_header: true
+            }
+          ])
+        end
+      end
+
+      context 'when section name includes $' do
+        let(:section_name) { 'my_$ection' }
+
+        it 'ignores the section' do
+          expect(convert_json("#{section_start}hello")).to eq([
+            {
+              offset: 0,
+              content: [{ text: "#{section_start.gsub("\033[0K", '')}hello" }]
+            }
+          ])
+        end
+      end
+
+      context 'when section name includes <' do
+        let(:section_name) { '<a_tag>' }
+
+        it 'ignores the section' do
+          expect(convert_json("#{section_start}hello")).to eq([
+            {
+              offset: 0,
+              content: [{ text: "#{section_start.gsub("\033[0K", '').gsub('<', '&lt;')}hello" }]
+            }
+          ])
+        end
+      end
+
+      it 'prevents XSS injection' do
+        trace = "#{section_start}section_end:1:2<script>alert('XSS Hack!');</script>#{section_end}"
+        expect(convert_json(trace)).to eq([
+          {
+            offset: 0,
+            content: [{ text: "section_end:1:2&lt;script>alert('XSS Hack!');&lt;/script>" }],
+            section: 'prepare-script',
+            section_header: true
+          },
+          {
+            offset: 95,
+            content: [],
+            section: 'prepare-script',
+            section_duration: '01:03'
+          },
+          {
+            offset: 95,
+            content: []
+          }
+        ])
+      end
+
+      context 'with nested section' do
+        let(:nested_section_name) { 'prepare-script-nested' }
+        let(:nested_section_duration) { 2.seconds }
+        let(:nested_section_start_time) { Time.new(2019, 9, 17).utc }
+        let(:nested_section_end_time) { nested_section_start_time + nested_section_duration }
+        let(:nested_section_start) { "section_start:#{nested_section_start_time.to_i}:#{nested_section_name}\r\033[0K"}
+        let(:nested_section_end) { "section_end:#{nested_section_end_time.to_i}:#{nested_section_name}\r\033[0K"}
+
+        it 'adds multiple sections to the lines inside the nested section' do
+          trace = "Hello#{section_start}foo#{nested_section_start}bar#{nested_section_end}baz#{section_end}world"
+
+          expect(convert_json(trace)).to eq([
+              {
+                offset: 0,
+                content: [{ text: 'Hello' }]
+              },
+              {
+                offset: 5,
+                content: [{ text: 'foo' }],
+                section: 'prepare-script',
+                section_header: true
+              },
+              {
+                offset: 52,
+                content: [{ text: 'bar' }],
+                section: 'prepare-script-nested',
+                section_header: true
+              },
+              {
+                offset: 106,
+                content: [],
+                section: 'prepare-script-nested',
+                section_duration: '00:02'
+              },
+              {
+                offset: 106,
+                content: [{ text: 'baz' }],
+                section: 'prepare-script'
+              },
+              {
+                offset: 158,
+                content: [],
+                section: 'prepare-script',
+                section_duration: '01:03'
+              },
+              {
+                offset: 158,
+                content: [{ text: 'world' }]
+              }
+            ])
+        end
+
+        it 'adds multiple sections to the lines inside the nested section and closes all sections together' do
+          trace = "Hello#{section_start}\e[91mfoo\e[0m#{nested_section_start}bar#{nested_section_end}#{section_end}"
+
+          expect(convert_json(trace)).to eq([
+              {
+                offset: 0,
+                content: [{ text: 'Hello' }]
+              },
+              {
+                offset: 5,
+                content: [{ text: 'foo', style: 'term-fg-l-red' }],
+                section: 'prepare-script',
+                section_header: true
+              },
+              {
+                offset: 61,
+                content: [{ text: 'bar' }],
+                section: 'prepare-script-nested',
+                section_header: true
+              },
+              {
+                offset: 115,
+                content: [],
+                section: 'prepare-script-nested',
+                section_duration: '00:02'
+              },
+              {
+                offset: 115,
+                content: [],
+                section: 'prepare-script',
+                section_duration: '01:03'
+              },
+              {
+                offset: 164,
+                content: []
+              }
+            ])
+        end
+      end
+    end
+
+    describe 'incremental updates' do
+      let(:pass1_stream) { StringIO.new(pre_text) }
+      let(:pass2_stream) { StringIO.new(pre_text + text) }
+      let(:pass1) { subject.convert(pass1_stream) }
+      let(:pass2) { subject.convert(pass2_stream, pass1.state) }
+
+      context 'with split word' do
+        let(:pre_text) { "\e[1mHello " }
+        let(:text) { "World" }
+
+        let(:lines) do
+          [
+            { offset: 0, content: [{ text: 'Hello World', style: 'term-bold' }] }
+          ]
+        end
+
+        it 'returns the full line' do
+          expect(pass2.lines).to eq(lines)
+          expect(pass2.append).to be_falsey
+        end
+      end
+
+      context 'with split word on second line' do
+        let(:pre_text) { "Good\nmorning " }
+        let(:text) { "World" }
+
+        let(:lines) do
+          [
+            { offset: 5, content: [{ text: 'morning World' }] }
+          ]
+        end
+
+        it 'returns all lines since last partially processed line' do
+          expect(pass2.lines).to eq(lines)
+          expect(pass2.append).to be_truthy
+        end
+      end
+
+      context 'with split sequence across multiple lines' do
+        let(:pre_text) { "\e[1mgood\nmorning\n" }
+        let(:text) { "\e[3mworld" }
+
+        let(:lines) do
+          [
+            { offset: 17, content: [{ text: 'world', style: 'term-bold term-italic' }] }
+          ]
+        end
+
+        it 'returns the full line' do
+          expect(pass2.lines).to eq(lines)
+          expect(pass2.append).to be_truthy
+        end
+      end
+
+      context 'with split partial sequence' do
+        let(:pre_text) { "hello\e" }
+        let(:text) { "[1m world" }
+
+        let(:lines) do
+          [
+            { offset: 0, content: [
+              { text: 'hello' },
+              { text: ' world', style: 'term-bold' }
+            ] }
+          ]
+        end
+
+        it 'returns the full line' do
+          expect(pass2.lines).to eq(lines)
+          expect(pass2.append).to be_falsey
+        end
+      end
+
+      context 'with split new line' do
+        let(:pre_text) { "hello\r" }
+        let(:text) { "\nworld" }
+
+        let(:lines) do
+          [
+            { offset: 0, content: [{ text: 'hello' }] },
+            { offset: 7, content: [{ text: 'world' }] }
+          ]
+        end
+
+        it 'returns the full line' do
+          expect(pass2.lines).to eq(lines)
+          expect(pass2.append).to be_falsey
+        end
+      end
+
+      context 'with split section' do
+        let(:section_name) { 'prepare-script' }
+        let(:section_duration) { 63.seconds }
+        let(:section_start_time) { Time.new(2019, 9, 17).utc }
+        let(:section_end_time) { section_start_time + section_duration }
+        let(:section_start) { "section_start:#{section_start_time.to_i}:#{section_name}\r\033[0K"}
+        let(:section_end) { "section_end:#{section_end_time.to_i}:#{section_name}\r\033[0K"}
+
+        context 'with split section body' do
+          let(:pre_text) { "#{section_start}this is a header\nand " }
+          let(:text) { "this\n is a body" }
+
+          let(:lines) do
+            [
+              {
+                offset: 61,
+                content: [{ text: 'and this' }],
+                section: 'prepare-script'
+              },
+              {
+                offset: 70,
+                content: [{ text: ' is a body' }],
+                section: 'prepare-script'
+              }
+            ]
+          end
+
+          it 'returns the full line' do
+            expect(pass2.lines).to eq(lines)
+            expect(pass2.append).to be_truthy
+          end
+        end
+
+        context 'with split section where header is also split' do
+          let(:pre_text) { "#{section_start}this is " }
+          let(:text) { "a header\nand body" }
+
+          let(:lines) do
+            [
+              {
+                offset: 0,
+                content: [{ text: 'this is a header' }],
+                section: 'prepare-script',
+                section_header: true
+              },
+              {
+                offset: 61,
+                content: [{ text: 'and body' }],
+                section: 'prepare-script'
+              }
+            ]
+          end
+
+          it 'returns the full line' do
+            expect(pass2.lines).to eq(lines)
+            expect(pass2.append).to be_falsey
+          end
+        end
+
+        context 'with split section end' do
+          let(:pre_text) { "#{section_start}this is a header\nthe" }
+          let(:text) { " body\nthe end#{section_end}" }
+
+          let(:lines) do
+            [
+              {
+                offset: 61,
+                content: [{ text: 'the body' }],
+                section: 'prepare-script'
+              },
+              {
+                offset: 70,
+                content: [{ text: 'the end' }],
+                section: 'prepare-script'
+              },
+              {
+                offset: 77,
+                content: [],
+                section: 'prepare-script',
+                section_duration: '01:03'
+              },
+              {
+                offset: 77,
+                content: []
+              }
+            ]
+          end
+
+          it 'returns the full line' do
+            expect(pass2.lines).to eq(lines)
+            expect(pass2.append).to be_truthy
+          end
+        end
+      end
+    end
+
+    describe 'trucates' do
+      let(:text) { "Hello World" }
+      let(:stream) { StringIO.new(text) }
+      let(:subject) { described_class.convert(stream) }
+
+      before do
+        stream.seek(3, IO::SEEK_SET)
+      end
+
+      it "returns truncated output" do
+        expect(subject.truncated).to be_truthy
+      end
+
+      it "does not append output" do
+        expect(subject.append).to be_falsey
+      end
+    end
+
+    def convert_json(data)
+      stream = StringIO.new(data)
+      subject.convert(stream).lines
+    end
+  end
+end