Commit 828d81ee authored by Kamil Trzciński's avatar Kamil Trzciński Committed by Rémy Coutable

Optimise trace handling code to use streaming instead of full read

parent 514dc1a0
...@@ -84,10 +84,10 @@ window.Build = (function() { ...@@ -84,10 +84,10 @@ window.Build = (function() {
var removeRefreshStatuses = ['success', 'failed', 'canceled', 'skipped']; var removeRefreshStatuses = ['success', 'failed', 'canceled', 'skipped'];
return $.ajax({ return $.ajax({
url: this.buildUrl, url: this.pageUrl + "/trace.json",
dataType: 'json', dataType: 'json',
success: function(buildData) { success: function(buildData) {
$('.js-build-output').html(buildData.trace_html); $('.js-build-output').html(buildData.html);
gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`); gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`);
if (window.location.hash === DOWN_BUILD_TRACE) { if (window.location.hash === DOWN_BUILD_TRACE) {
$("html,body").scrollTop(this.$buildTrace.height()); $("html,body").scrollTop(this.$buildTrace.height());
......
...@@ -31,25 +31,25 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -31,25 +31,25 @@ class Projects::BuildsController < Projects::ApplicationController
@builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC') @builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC')
@builds = @builds.where("id not in (?)", @build.id) @builds = @builds.where("id not in (?)", @build.id)
@pipeline = @build.pipeline @pipeline = @build.pipeline
respond_to do |format|
format.html
format.json do
render json: {
id: @build.id,
status: @build.status,
trace_html: @build.trace_html
}
end
end
end end
def trace def trace
respond_to do |format| build.trace.read do |stream|
format.json do respond_to do |format|
state = params[:state].presence format.json do
render json: @build.trace_with_state(state: state). result = {
merge!(id: @build.id, status: @build.status) id: @build.id, status: @build.status, complete: @build.complete?
}
if stream.valid?
stream.limit
state = params[:state].presence
trace = stream.html_with_state(state)
result.merge!(trace.to_h)
end
render json: result
end
end end
end end
end end
...@@ -86,10 +86,12 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -86,10 +86,12 @@ class Projects::BuildsController < Projects::ApplicationController
end end
def raw def raw
if @build.has_trace_file? build.trace.read do |stream|
send_file @build.trace_file_path, type: 'text/plain; charset=utf-8', disposition: 'inline' if stream.file?
else send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline'
render_404 else
render_404
end
end end
end end
......
...@@ -171,19 +171,6 @@ module Ci ...@@ -171,19 +171,6 @@ module Ci
latest_builds.where('stage_idx < ?', stage_idx) latest_builds.where('stage_idx < ?', stage_idx)
end end
def trace_html(**args)
trace_with_state(**args)[:html] || ''
end
def trace_with_state(state: nil, last_lines: nil)
trace_ansi = trace(last_lines: last_lines)
if trace_ansi.present?
Ci::Ansi2html.convert(trace_ansi, state)
else
{}
end
end
def timeout def timeout
project.build_timeout project.build_timeout
end end
...@@ -244,136 +231,35 @@ module Ci ...@@ -244,136 +231,35 @@ module Ci
end end
def update_coverage def update_coverage
coverage = extract_coverage(trace, coverage_regex) coverage = trace.extract_coverage(coverage_regex)
update_attributes(coverage: coverage) if coverage.present? update_attributes(coverage: coverage) if coverage.present?
end end
def extract_coverage(text, regex) def trace
return unless regex Gitlab::Ci::Trace.new(self)
matches = text.scan(Regexp.new(regex)).last
matches = matches.last if matches.is_a?(Array)
coverage = matches.gsub(/\d+(\.\d+)?/).first
if coverage.present?
coverage.to_f
end
rescue
# if bad regex or something goes wrong we dont want to interrupt transition
# so we just silentrly ignore error for now
end
def has_trace_file?
File.exist?(path_to_trace) || has_old_trace_file?
end end
def has_trace? def has_trace?
raw_trace.present? trace.exist?
end end
def raw_trace(last_lines: nil) def trace=(data)
if File.exist?(trace_file_path) raise NotImplementedError
Gitlab::Ci::TraceReader.new(trace_file_path).
read(last_lines: last_lines)
else
# backward compatibility
read_attribute :trace
end
end end
## def old_trace
# Deprecated read_attribute(:trace)
#
# This is a hotfix for CI build data integrity, see #4246
def has_old_trace_file?
project.ci_id && File.exist?(old_path_to_trace)
end
def trace(last_lines: nil)
hide_secrets(raw_trace(last_lines: last_lines))
end end
def trace_length def erase_old_trace!
if raw_trace write_attribute(:trace, nil)
raw_trace.bytesize save
else
0
end
end
def trace=(trace)
recreate_trace_dir
trace = hide_secrets(trace)
File.write(path_to_trace, trace)
end
def recreate_trace_dir
unless Dir.exist?(dir_to_trace)
FileUtils.mkdir_p(dir_to_trace)
end
end
private :recreate_trace_dir
def append_trace(trace_part, offset)
recreate_trace_dir
touch if needs_touch?
trace_part = hide_secrets(trace_part)
File.truncate(path_to_trace, offset) if File.exist?(path_to_trace)
File.open(path_to_trace, 'ab') do |f|
f.write(trace_part)
end
end end
def needs_touch? def needs_touch?
Time.now - updated_at > 15.minutes.to_i Time.now - updated_at > 15.minutes.to_i
end end
def trace_file_path
if has_old_trace_file?
old_path_to_trace
else
path_to_trace
end
end
def dir_to_trace
File.join(
Settings.gitlab_ci.builds_path,
created_at.utc.strftime("%Y_%m"),
project.id.to_s
)
end
def path_to_trace
"#{dir_to_trace}/#{id}.log"
end
##
# Deprecated
#
# This is a hotfix for CI build data integrity, see #4246
# Should be removed in 8.4, after CI files migration has been done.
#
def old_dir_to_trace
File.join(
Settings.gitlab_ci.builds_path,
created_at.utc.strftime("%Y_%m"),
project.ci_id.to_s
)
end
##
# Deprecated
#
# This is a hotfix for CI build data integrity, see #4246
# Should be removed in 8.4, after CI files migration has been done.
#
def old_path_to_trace
"#{old_dir_to_trace}/#{id}.log"
end
## ##
# Deprecated # Deprecated
# #
...@@ -555,6 +441,15 @@ module Ci ...@@ -555,6 +441,15 @@ module Ci
options[:dependencies]&.empty? options[:dependencies]&.empty?
end end
def hide_secrets(trace)
return unless trace
trace = trace.dup
Ci::MaskSecret.mask!(trace, project.runners_token) if project
Ci::MaskSecret.mask!(trace, token)
trace
end
private private
def update_artifacts_size def update_artifacts_size
...@@ -566,7 +461,7 @@ module Ci ...@@ -566,7 +461,7 @@ module Ci
end end
def erase_trace! def erase_trace!
self.trace = nil trace.erase!
end end
def update_erased!(user = nil) def update_erased!(user = nil)
...@@ -628,15 +523,6 @@ module Ci ...@@ -628,15 +523,6 @@ module Ci
pipeline.config_processor.build_attributes(name) pipeline.config_processor.build_attributes(name)
end end
def hide_secrets(trace)
return unless trace
trace = trace.dup
Ci::MaskSecret.mask!(trace, project.runners_token) if project
Ci::MaskSecret.mask!(trace, token)
trace
end
def update_project_statistics def update_project_statistics
return unless project return unless project
......
...@@ -137,6 +137,6 @@ ...@@ -137,6 +137,6 @@
- if build.has_trace? - if build.has_trace?
%td{ colspan: "2", style: "font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;padding:0 0 15px;" } %td{ colspan: "2", style: "font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;padding:0 0 15px;" }
%pre{ style: "font-family:Monaco,'Lucida Console','Courier New',Courier,monospace;background-color:#fafafa;border-radius:3px;overflow:hidden;white-space:pre-wrap;word-break:break-all;font-size:13px;line-height:1.4;padding:12px;color:#333333;margin:0;" } %pre{ style: "font-family:Monaco,'Lucida Console','Courier New',Courier,monospace;background-color:#fafafa;border-radius:3px;overflow:hidden;white-space:pre-wrap;word-break:break-all;font-size:13px;line-height:1.4;padding:12px;color:#333333;margin:0;" }
= build.trace_html(last_lines: 10).html_safe = build.trace.html(last_lines: 10).html_safe
- else - else
%td{ colspan: "2" } %td{ colspan: "2" }
...@@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>. ...@@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
Stage: <%= build.stage %> Stage: <%= build.stage %>
Name: <%= build.name %> Name: <%= build.name %>
<% if build.has_trace? -%> <% if build.has_trace? -%>
Trace: <%= build.trace_with_state(last_lines: 10)[:text] %> Trace: <%= build.trace.raw(last_lines: 10) %>
<% end -%> <% end -%>
<% end -%> <% end -%>
...@@ -68,7 +68,7 @@ ...@@ -68,7 +68,7 @@
- elsif @build.runner - elsif @build.runner
\##{@build.runner.id} \##{@build.runner.id}
.btn-group.btn-group-justified{ role: :group } .btn-group.btn-group-justified{ role: :group }
- if @build.has_trace_file? - if @build.has_trace?
= link_to 'Raw', raw_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' = link_to 'Raw', raw_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default'
- if @build.active? - if @build.active?
= link_to "Cancel", cancel_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post = link_to "Cancel", cancel_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post
......
...@@ -130,7 +130,7 @@ class Gitlab::Seeder::Pipelines ...@@ -130,7 +130,7 @@ class Gitlab::Seeder::Pipelines
def setup_build_log(build) def setup_build_log(build)
if %w(running success failed).include?(build.status) if %w(running success failed).include?(build.status)
build.trace = FFaker::Lorem.paragraphs(6).join("\n\n") build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n"))
end end
end end
......
...@@ -22,9 +22,9 @@ class Spinach::Features::ProjectBuildsSummary < Spinach::FeatureSteps ...@@ -22,9 +22,9 @@ class Spinach::Features::ProjectBuildsSummary < Spinach::FeatureSteps
end end
step 'recent build has been erased' do step 'recent build has been erased' do
expect(@build).not_to have_trace
expect(@build.artifacts_file.exists?).to be_falsy expect(@build.artifacts_file.exists?).to be_falsy
expect(@build.artifacts_metadata.exists?).to be_falsy expect(@build.artifacts_metadata.exists?).to be_falsy
expect(@build.trace).to be_empty
end end
step 'recent build summary does not have artifacts widget' do step 'recent build summary does not have artifacts widget' do
......
...@@ -47,7 +47,7 @@ module SharedBuilds ...@@ -47,7 +47,7 @@ module SharedBuilds
end end
step 'recent build has a build trace' do step 'recent build has a build trace' do
@build.trace = 'job trace' @build.trace.set('job trace')
end end
step 'download of build artifacts archive starts' do step 'download of build artifacts archive starts' do
......
...@@ -118,7 +118,7 @@ module API ...@@ -118,7 +118,7 @@ module API
content_type 'text/plain' content_type 'text/plain'
env['api.format'] = :binary env['api.format'] = :binary
trace = build.trace trace = build.trace.raw
body trace body trace
end end
......
...@@ -115,7 +115,7 @@ module API ...@@ -115,7 +115,7 @@ module API
put '/:id' do put '/:id' do
job = authenticate_job! job = authenticate_job!
job.update_attributes(trace: params[:trace]) if params[:trace] job.trace.set(params[:trace]) if params[:trace]
Gitlab::Metrics.add_event(:update_build, Gitlab::Metrics.add_event(:update_build,
project: job.project.path_with_namespace) project: job.project.path_with_namespace)
...@@ -145,16 +145,14 @@ module API ...@@ -145,16 +145,14 @@ module API
content_range = request.headers['Content-Range'] content_range = request.headers['Content-Range']
content_range = content_range.split('-') content_range = content_range.split('-')
current_length = job.trace_length stream_size = job.trace.append(request.body.read, content_range[0].to_i)
unless current_length == content_range[0].to_i if stream_size < 0
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
end end
job.append_trace(request.body.read, content_range[0].to_i)
status 202 status 202
header 'Job-Status', job.status header 'Job-Status', job.status
header 'Range', "0-#{job.trace_length}" header 'Range', "0-#{stream_size}"
end end
desc 'Authorize artifacts uploading for job' do desc 'Authorize artifacts uploading for job' do
......
...@@ -120,7 +120,7 @@ module API ...@@ -120,7 +120,7 @@ module API
content_type 'text/plain' content_type 'text/plain'
env['api.format'] = :binary env['api.format'] = :binary
trace = build.trace trace = build.trace.raw
body trace body trace
end end
......
...@@ -132,34 +132,54 @@ module Ci ...@@ -132,34 +132,54 @@ module Ci
STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
def convert(raw, new_state) def convert(stream, new_state)
reset_state reset_state
restore_state(raw, new_state) if new_state.present? restore_state(new_state, stream) if new_state.present?
start = @offset append = false
ansi = raw[@offset..-1] truncated = false
cur_offset = stream.tell
if cur_offset > @offset
@offset = cur_offset
truncated = true
else
stream.seek(@offset)
append = @offset > 0
end
start_offset = @offset
open_new_tag open_new_tag
s = StringScanner.new(ansi) stream.each_line do |line|
until s.eos? s = StringScanner.new(line)
if s.scan(/\e([@-_])(.*?)([@-~])/) until s.eos?
handle_sequence(s) if s.scan(/\e([@-_])(.*?)([@-~])/)
elsif s.scan(/\e(([@-_])(.*?)?)?$/) handle_sequence(s)
break elsif s.scan(/\e(([@-_])(.*?)?)?$/)
elsif s.scan(/</) break
@out << '&lt;' elsif s.scan(/</)
elsif s.scan(/\r?\n/) @out << '&lt;'
@out << '<br>' elsif s.scan(/\r?\n/)
else @out << '<br>'
@out << s.scan(/./m) else
@out << s.scan(/./m)
end
@offset += s.matched_size
end end
@offset += s.matched_size
end end
close_open_tags() close_open_tags()
{ state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } OpenStruct.new(
html: @out,
state: state,
append: append,
truncated: truncated,
offset: start_offset,
size: stream.tell - start_offset,
total: stream.size
)
end end
def handle_sequence(s) def handle_sequence(s)
...@@ -240,10 +260,10 @@ module Ci ...@@ -240,10 +260,10 @@ module Ci
Base64.urlsafe_encode64(state.to_json) Base64.urlsafe_encode64(state.to_json)
end end
def restore_state(raw, new_state) def restore_state(new_state, stream)
state = Base64.urlsafe_decode64(new_state) state = Base64.urlsafe_decode64(new_state)
state = JSON.parse(state, symbolize_names: true) state = JSON.parse(state, symbolize_names: true)
return if state[:offset].to_i > raw.length return if state[:offset].to_i > stream.size
STATE_PARAMS.each do |param| STATE_PARAMS.each do |param|
send("#{param}=".to_sym, state[param]) send("#{param}=".to_sym, state[param])
......
...@@ -61,7 +61,7 @@ module Ci ...@@ -61,7 +61,7 @@ module Ci
update_runner_info update_runner_info
build.update_attributes(trace: params[:trace]) if params[:trace] build.trace.set(params[:trace]) if params[:trace]
Gitlab::Metrics.add_event(:update_build, Gitlab::Metrics.add_event(:update_build,
project: build.project.path_with_namespace) project: build.project.path_with_namespace)
...@@ -92,16 +92,14 @@ module Ci ...@@ -92,16 +92,14 @@ module Ci
content_range = request.headers['Content-Range'] content_range = request.headers['Content-Range']
content_range = content_range.split('-') content_range = content_range.split('-')
current_length = build.trace_length stream_size = build.trace.append(request.body.read, content_range[0].to_i)
unless current_length == content_range[0].to_i if stream_size < 0
return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
end end
build.append_trace(request.body.read, content_range[0].to_i)
status 202 status 202
header 'Build-Status', build.status header 'Build-Status', build.status
header 'Range', "0-#{build.trace_length}" header 'Range', "0-#{stream_size}"
end end
# Authorize artifacts uploading for build - Runners only # Authorize artifacts uploading for build - Runners only
......
module Gitlab
module Ci
class Trace
attr_reader :job
delegate :old_trace, to: :job
def initialize(job)
@job = job
end
def html(last_lines: nil)
read do |stream|
stream.html(last_lines: last_lines)
end
end
def raw(last_lines: nil)
read do |stream|
stream.raw(last_lines: last_lines)
end
end
def extract_coverage(regex)
read do |stream|
stream.extract_coverage(regex)
end
end
def set(data)
write do |stream|
data = job.hide_secrets(data)
stream.set(data)
end
end
def append(data, offset)
write do |stream|
current_length = stream.size
return -current_length unless current_length == offset
data = job.hide_secrets(data)
stream.append(data, offset)
stream.size
end
end
def exist?
current_path.present? || old_trace.present?
end
def read
stream = Gitlab::Ci::Trace::Stream.new do
if current_path
File.open(current_path, "rb")
elsif old_trace
StringIO.new(old_trace)
end
end
yield stream
ensure
stream&.close
end
def write
stream = Gitlab::Ci::Trace::Stream.new do
File.open(ensure_path, "a+b")
end
yield(stream).tap do
job.touch if job.needs_touch?
end
ensure
stream&.close
end
def erase!
paths.each do |trace_path|
FileUtils.rm(trace_path, force: true)
end
job.erase_old_trace!
end
private
def ensure_path
return current_path if current_path
ensure_directory
default_path
end
def ensure_directory
unless Dir.exist?(default_directory)
FileUtils.mkdir_p(default_directory)
end
end
def current_path
@current_path ||= paths.find do |trace_path|
File.exist?(trace_path)
end
end
def paths
[
default_path,
deprecated_path
].compact
end
def default_directory
File.join(
Settings.gitlab_ci.builds_path,
job.created_at.utc.strftime("%Y_%m"),
job.project_id.to_s
)
end
def default_path
File.join(default_directory, "#{job.id}.log")
end
def deprecated_path
File.join(
Settings.gitlab_ci.builds_path,
job.created_at.utc.strftime("%Y_%m"),
job.project.ci_id.to_s,
"#{job.id}.log"
) if job.project&.ci_id
end
end
end
end
module Gitlab
module Ci
class Trace
# This was inspired from: http://stackoverflow.com/a/10219411/1520132
class Stream
BUFFER_SIZE = 4096
LIMIT_SIZE = 50.kilobytes
attr_reader :stream
delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
delegate :valid?, to: :stream, as: :present?, allow_nil: true
def initialize
@stream = yield
end
def valid?
self.stream.present?
end
def file?
self.path.present?
end
def limit(last_bytes = LIMIT_SIZE)
stream_size = size
if stream_size < last_bytes
last_bytes = stream_size
end
stream.seek(-last_bytes, IO::SEEK_END)
end
def append(data, offset)
stream.truncate(offset)
stream.seek(0, IO::SEEK_END)
stream.write(data)
stream.flush()
end
def set(data)
truncate(0)
stream.write(data)
stream.flush()
end
def raw(last_lines: nil)
return unless valid?
if last_lines.to_i > 0
read_last_lines(last_lines)
else
stream.read
end
end
def html_with_state(state = nil)
::Ci::Ansi2html.convert(stream, state)
end
def html(last_lines: nil)
text = raw(last_lines: last_lines)
stream = StringIO.new(text)
::Ci::Ansi2html.convert(stream).html
end
def extract_coverage(regex)
return unless valid?
return unless regex
regex = Regexp.new(regex)
match = ""
stream.each_line do |line|
matches = line.scan(regex)
next unless matches.is_a?(Array)
match = matches.flatten.last
coverage = match.gsub(/\d+(\.\d+)?/).first
return coverage.to_f if coverage.present?
end
rescue
# if bad regex or something goes wrong we dont want to interrupt transition
# so we just silentrly ignore error for now
end
private
def read_last_lines(last_lines)
chunks = []
pos = lines = 0
max = stream.size
# We want an extra line to make sure fist line has full contents
while lines <= last_lines && pos < max
pos += BUFFER_SIZE
buf =
if pos <= max
stream.seek(-pos, IO::SEEK_END)
stream.read(BUFFER_SIZE)
else # Reached the head, read only left
stream.seek(0)
stream.read(BUFFER_SIZE - (pos - max))
end
lines += buf.count("\n")
chunks.unshift(buf)
end
chunks.join.lines.last(last_lines).join
.force_encoding(Encoding.default_external)
end
end
end
end
end
module Gitlab
module Ci
# This was inspired from: http://stackoverflow.com/a/10219411/1520132
class TraceReader
BUFFER_SIZE = 4096
attr_accessor :path, :buffer_size
def initialize(new_path, buffer_size: BUFFER_SIZE)
self.path = new_path
self.buffer_size = Integer(buffer_size)
end
def read(last_lines: nil)
if last_lines
read_last_lines(last_lines)
else
File.read(path)
end
end
def read_last_lines(max_lines)
File.open(path) do |file|
chunks = []
pos = lines = 0
max = file.size
# We want an extra line to make sure fist line has full contents
while lines <= max_lines && pos < max
pos += buffer_size
buf = if pos <= max
file.seek(-pos, IO::SEEK_END)
file.read(buffer_size)
else # Reached the head, read only left
file.seek(0)
file.read(buffer_size - (pos - max))
end
lines += buf.count("\n")
chunks.unshift(buf)
end
chunks.join.lines.last(max_lines).join
.force_encoding(Encoding.default_external)
end
end
end
end
end
...@@ -111,7 +111,7 @@ FactoryGirl.define do ...@@ -111,7 +111,7 @@ FactoryGirl.define do
trait :trace do trait :trace do
after(:create) do |build, evaluator| after(:create) do |build, evaluator|
build.trace = 'BUILD TRACE' build.trace.set('BUILD TRACE')
end end
end end
......
...@@ -205,21 +205,13 @@ feature 'Builds', :feature do ...@@ -205,21 +205,13 @@ feature 'Builds', :feature do
it 'loads job trace' do it 'loads job trace' do
expect(page).to have_content 'BUILD TRACE' expect(page).to have_content 'BUILD TRACE'
build.append_trace(' and more trace', 11) build.trace.write do |stream|
stream.append(' and more trace', 11)
end
expect(page).to have_content 'BUILD TRACE and more trace' expect(page).to have_content 'BUILD TRACE and more trace'
end end
end end
context 'when build does not have an initial trace' do
let(:build) { create(:ci_build, pipeline: pipeline) }
it 'loads new trace' do
build.append_trace('build trace', 0)
expect(page).to have_content 'build trace'
end
end
end end
feature 'Variables' do feature 'Variables' do
...@@ -390,7 +382,7 @@ feature 'Builds', :feature do ...@@ -390,7 +382,7 @@ feature 'Builds', :feature do
it 'sends the right headers' do it 'sends the right headers' do
expect(page.status_code).to eq(200) expect(page.status_code).to eq(200)
expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8') expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
expect(page.response_headers['X-Sendfile']).to eq(build.path_to_trace) expect(page.response_headers['X-Sendfile']).to eq(build.trace.send(:current_path))
end end
end end
...@@ -409,43 +401,24 @@ feature 'Builds', :feature do ...@@ -409,43 +401,24 @@ feature 'Builds', :feature do
context 'storage form' do context 'storage form' do
let(:existing_file) { Tempfile.new('existing-trace-file').path } let(:existing_file) { Tempfile.new('existing-trace-file').path }
let(:non_existing_file) do
file = Tempfile.new('non-existing-trace-file')
path = file.path
file.unlink
path
end
context 'when build has trace in file' do before do
before do Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
build.run!
visit namespace_project_build_path(project.namespace, project, build)
allow_any_instance_of(Project).to receive(:ci_id).and_return(nil) build.run!
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file)
page.within('.js-build-sidebar') { click_link 'Raw' } allow_any_instance_of(Gitlab::Ci::Trace).to receive(:paths)
end .and_return(paths)
it 'sends the right headers' do visit namespace_project_build_path(project.namespace, project, build)
expect(page.status_code).to eq(200)
expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8')
expect(page.response_headers['X-Sendfile']).to eq(existing_file)
end
end end
context 'when build has trace in old file' do context 'when build has trace in file' do
before do let(:paths) do
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') [existing_file]
build.run! end
visit namespace_project_build_path(project.namespace, project, build)
allow_any_instance_of(Project).to receive(:ci_id).and_return(999)
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(existing_file)
before do
page.within('.js-build-sidebar') { click_link 'Raw' } page.within('.js-build-sidebar') { click_link 'Raw' }
end end
...@@ -457,20 +430,10 @@ feature 'Builds', :feature do ...@@ -457,20 +430,10 @@ feature 'Builds', :feature do
end end
context 'when build has trace in DB' do context 'when build has trace in DB' do
before do let(:paths) { [] }
Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile')
build.run!
visit namespace_project_build_path(project.namespace, project, build)
allow_any_instance_of(Project).to receive(:ci_id).and_return(nil)
allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file)
allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file)
page.within('.js-build-sidebar') { click_link 'Raw' }
end
it 'sends the right headers' do it 'sends the right headers' do
expect(page.status_code).to eq(404) expect(page.status_code).not_to have_link('Raw')
end end
end end
end end
......
...@@ -72,12 +72,14 @@ describe('Build', () => { ...@@ -72,12 +72,14 @@ describe('Build', () => {
it('displays the initial build trace', () => { it('displays the initial build trace', () => {
expect($.ajax.calls.count()).toBe(1); expect($.ajax.calls.count()).toBe(1);
const [{ url, dataType, success, context }] = $.ajax.calls.argsFor(0); const [{ url, dataType, success, context }] = $.ajax.calls.argsFor(0);
expect(url).toBe(`${BUILD_URL}.json`); expect(url).toBe(
`${BUILD_URL}/trace.json`,
);
expect(dataType).toBe('json'); expect(dataType).toBe('json');
expect(success).toEqual(jasmine.any(Function)); expect(success).toEqual(jasmine.any(Function));
spyOn(gl.utils, 'setCiStatusFavicon').and.callFake(() => {}); spyOn(gl.utils, 'setCiStatusFavicon').and.callFake(() => {});
success.call(context, { trace_html: '<span>Example</span>', status: 'running' }); success.call(context, { html: '<span>Example</span>', status: 'running' });
expect($('#build-trace .js-build-output').text()).toMatch(/Example/); expect($('#build-trace .js-build-output').text()).toMatch(/Example/);
}); });
......
This diff is collapsed.
require 'spec_helper'
describe Gitlab::Ci::Trace::Stream do
describe 'delegates' do
subject { described_class.new { nil } }
it { is_expected.to delegate_method(:close).to(:stream) }
it { is_expected.to delegate_method(:tell).to(:stream) }
it { is_expected.to delegate_method(:seek).to(:stream) }
it { is_expected.to delegate_method(:size).to(:stream) }
it { is_expected.to delegate_method(:path).to(:stream) }
it { is_expected.to delegate_method(:truncate).to(:stream) }
it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) }
it { is_expected.to delegate_method(:file?).to(:path).as(:present?) }
end
describe '#limit' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
it 'if size is larger we start from beggining' do
stream.limit(10)
expect(stream.tell).to eq(0)
end
it 'if size is smaller we start from the end' do
stream.limit(2)
expect(stream.tell).to eq(6)
end
end
describe '#append' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
it "truncates and append content" do
stream.append("89", 4)
stream.seek(0)
expect(stream.size).to eq(6)
expect(stream.raw).to eq("123489")
end
end
describe '#set' do
let(:stream) do
described_class.new do
StringIO.new("12345678")
end
end
before do
stream.set("8901")
end
it "overwrite content" do
stream.seek(0)
expect(stream.size).to eq(4)
expect(stream.raw).to eq("8901")
end
end
describe '#raw' do
let(:path) { __FILE__ }
let(:lines) { File.readlines(path) }
let(:stream) do
described_class.new do
File.open(path)
end
end
it 'returns all contents if last_lines is not specified' do
result = stream.raw
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
context 'limit max lines' do
before do
# specifying BUFFER_SIZE forces to seek backwards
allow(described_class).to receive(:BUFFER_SIZE)
.and_return(2)
end
it 'returns last few lines' do
result = stream.raw(last_lines: 2)
expect(result).to eq(lines.last(2).join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'returns everything if trying to get too many lines' do
result = stream.raw(last_lines: lines.size * 2)
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
end
end
describe '#html_with_state' do
let(:stream) do
described_class.new do
StringIO.new("1234")
end
end
it 'returns html content with state' do
result = stream.html_with_state
expect(result.html).to eq("1234")
end
context 'follow-up state' do
let!(:last_result) { stream.html_with_state }
before do
stream.append("5678", 4)
stream.seek(0)
end
it "returns appended trace" do
result = stream.html_with_state(last_result.state)
expect(result.append).to be_truthy
expect(result.html).to eq("5678")
end
end
end
describe '#html' do
let(:stream) do
described_class.new do
StringIO.new("12\n34\n56")
end
end
it "returns html" do
expect(stream.html).to eq("12<br>34<br>56")
end
it "returns html for last line only" do
expect(stream.html(last_lines: 1)).to eq("56")
end
end
describe '#extract_coverage' do
let(:stream) do
described_class.new do
StringIO.new(data)
end
end
subject { stream.extract_coverage(regex) }
context 'valid content & regex' do
let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to eq(98.29) }
end
context 'valid content & bad regex' do
let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' }
let(:regex) { 'very covered' }
it { is_expected.to be_nil }
end
context 'no coverage content & regex' do
let(:data) { 'No coverage for today :sad:' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to be_nil }
end
context 'multiple results in content & regex' do
let(:data) { ' (98.39%) covered. (98.29%) covered' }
let(:regex) { '\(\d+.\d+\%\) covered' }
it { is_expected.to eq(98.29) }
end
context 'using a regex capture' do
let(:data) { 'TOTAL 9926 3489 65%' }
let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' }
it { is_expected.to eq(65) }
end
end
end
require 'spec_helper'
describe Gitlab::Ci::TraceReader do
let(:path) { __FILE__ }
let(:lines) { File.readlines(path) }
let(:bytesize) { lines.sum(&:bytesize) }
it 'returns last few lines' do
10.times do
subject = build_subject
last_lines = random_lines
expected = lines.last(last_lines).join
result = subject.read(last_lines: last_lines)
expect(result).to eq(expected)
expect(result.encoding).to eq(Encoding.default_external)
end
end
it 'returns everything if trying to get too many lines' do
result = build_subject.read(last_lines: lines.size * 2)
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'returns all contents if last_lines is not specified' do
result = build_subject.read
expect(result).to eq(lines.join)
expect(result.encoding).to eq(Encoding.default_external)
end
it 'raises an error if not passing an integer for last_lines' do
expect do
build_subject.read(last_lines: lines)
end.to raise_error(ArgumentError)
end
def random_lines
Random.rand(lines.size) + 1
end
def random_buffer
Random.rand(bytesize) + 1
end
def build_subject
described_class.new(__FILE__, buffer_size: random_buffer)
end
end
require 'spec_helper'
describe Gitlab::Ci::Trace do
let(:build) { create(:ci_build) }
let(:trace) { described_class.new(build) }
describe "associations" do
it { expect(trace).to respond_to(:job) }
it { expect(trace).to delegate_method(:old_trace).to(:job) }
end
describe '#html' do
before do
trace.set("12\n34")
end
it "returns formatted html" do
expect(trace.html).to eq("12<br>34")
end
it "returns last line of formatted html" do
expect(trace.html(last_lines: 1)).to eq("34")
end
end
describe '#raw' do
before do
trace.set("12\n34")
end
it "returns raw output" do
expect(trace.raw).to eq("12\n34")
end
it "returns last line of raw output" do
expect(trace.raw(last_lines: 1)).to eq("34")
end
end
describe '#extract_coverage' do
let(:regex) { '\(\d+.\d+\%\) covered' }
before do
trace.set('Coverage 1033 / 1051 LOC (98.29%) covered')
end
it "returns valid coverage" do
expect(trace.extract_coverage(regex)).to eq(98.29)
end
end
describe '#set' do
before do
trace.set("12")
end
it "returns trace" do
expect(trace.raw).to eq("12")
end
context 'overwrite trace' do
before do
trace.set("34")
end
it "returns new trace" do
expect(trace.raw).to eq("34")
end
end
context 'runners token' do
let(:token) { 'my_secret_token' }
before do
build.project.update(runners_token: token)
trace.set(token)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
context 'hides build token' do
let(:token) { 'my_secret_token' }
before do
build.update(token: token)
trace.set(token)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
end
describe '#append' do
before do
trace.set("1234")
end
it "returns correct trace" do
expect(trace.append("56", 4)).to eq(6)
expect(trace.raw).to eq("123456")
end
context 'tries to append trace at different offset' do
it "fails with append" do
expect(trace.append("56", 2)).to eq(-4)
expect(trace.raw).to eq("1234")
end
end
context 'runners token' do
let(:token) { 'my_secret_token' }
before do
build.project.update(runners_token: token)
trace.append(token, 0)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
context 'build token' do
let(:token) { 'my_secret_token' }
before do
build.update(token: token)
trace.append(token, 0)
end
it "hides token" do
expect(trace.raw).not_to include(token)
end
end
end
describe 'trace handling' do
context 'trace does not exist' do
it { expect(trace.exist?).to be(false) }
end
context 'new trace path is used' do
before do
trace.send(:ensure_directory)
File.open(trace.send(:default_path), "w") do |file|
file.write("data")
end
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
end
context 'deprecated path' do
let(:path) { trace.send(:deprecated_path) }
context 'with valid ci_id' do
before do
build.project.update(ci_id: 1000)
FileUtils.mkdir_p(File.dirname(path))
File.open(path, "w") do |file|
file.write("data")
end
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
end
context 'without valid ci_id' do
it "does not return deprecated path" do
expect(path).to be_nil
end
end
end
context 'stored in database' do
before do
build.send(:write_attribute, :trace, "data")
end
it "trace exist" do
expect(trace.exist?).to be(true)
end
it "can be erased" do
trace.erase!
expect(trace.exist?).to be(false)
end
it "returns database data" do
expect(trace.raw).to eq("data")
end
end
end
end
This diff is collapsed.
...@@ -320,7 +320,7 @@ describe API::Jobs, api: true do ...@@ -320,7 +320,7 @@ describe API::Jobs, api: true do
context 'authorized user' do context 'authorized user' do
it 'returns specific job trace' do it 'returns specific job trace' do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(response.body).to eq(build.trace) expect(response.body).to eq(build.trace.raw)
end end
end end
...@@ -408,7 +408,7 @@ describe API::Jobs, api: true do ...@@ -408,7 +408,7 @@ describe API::Jobs, api: true do
it 'erases job content' do it 'erases job content' do
expect(response).to have_http_status(201) expect(response).to have_http_status(201)
expect(build.trace).to be_empty expect(build).not_to have_trace
expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_file.exists?).to be_falsy
expect(build.artifacts_metadata.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy
end end
......
...@@ -592,7 +592,7 @@ describe API::Runner do ...@@ -592,7 +592,7 @@ describe API::Runner do
update_job(trace: 'BUILD TRACE UPDATED') update_job(trace: 'BUILD TRACE UPDATED')
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(job.reload.trace).to eq 'BUILD TRACE UPDATED' expect(job.reload.trace.raw).to eq 'BUILD TRACE UPDATED'
end end
end end
...@@ -600,7 +600,7 @@ describe API::Runner do ...@@ -600,7 +600,7 @@ describe API::Runner do
it 'does not override trace information' do it 'does not override trace information' do
update_job update_job
expect(job.reload.trace).to eq 'BUILD TRACE' expect(job.reload.trace.raw).to eq 'BUILD TRACE'
end end
end end
...@@ -631,7 +631,7 @@ describe API::Runner do ...@@ -631,7 +631,7 @@ describe API::Runner do
context 'when request is valid' do context 'when request is valid' do
it 'gets correct response' do it 'gets correct response' do
expect(response.status).to eq 202 expect(response.status).to eq 202
expect(job.reload.trace).to eq 'BUILD TRACE appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Job-Status' expect(response.header).to have_key 'Job-Status'
end end
...@@ -642,7 +642,7 @@ describe API::Runner do ...@@ -642,7 +642,7 @@ describe API::Runner do
it "changes the job's trace" do it "changes the job's trace" do
patch_the_trace patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end end
context 'when Runner makes a force-patch' do context 'when Runner makes a force-patch' do
...@@ -651,7 +651,7 @@ describe API::Runner do ...@@ -651,7 +651,7 @@ describe API::Runner do
it "doesn't change the build.trace" do it "doesn't change the build.trace" do
force_patch_the_trace force_patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
end end
end end
end end
...@@ -664,7 +664,7 @@ describe API::Runner do ...@@ -664,7 +664,7 @@ describe API::Runner do
it 'changes the job.trace' do it 'changes the job.trace' do
patch_the_trace patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end end
context 'when Runner makes a force-patch' do context 'when Runner makes a force-patch' do
...@@ -673,7 +673,7 @@ describe API::Runner do ...@@ -673,7 +673,7 @@ describe API::Runner do
it "doesn't change the job.trace" do it "doesn't change the job.trace" do
force_patch_the_trace force_patch_the_trace
expect(job.reload.trace).to eq 'BUILD TRACE appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
end end
end end
end end
...@@ -698,7 +698,7 @@ describe API::Runner do ...@@ -698,7 +698,7 @@ describe API::Runner do
it 'gets correct response' do it 'gets correct response' do
expect(response.status).to eq 202 expect(response.status).to eq 202
expect(job.reload.trace).to eq 'BUILD TRACE appended' expect(job.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Job-Status' expect(response.header).to have_key 'Job-Status'
end end
...@@ -738,9 +738,11 @@ describe API::Runner do ...@@ -738,9 +738,11 @@ describe API::Runner do
def patch_the_trace(content = ' appended', request_headers = nil) def patch_the_trace(content = ' appended', request_headers = nil)
unless request_headers unless request_headers
offset = job.trace_length job.trace.read do |stream|
limit = offset + content.length - 1 offset = stream.size
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
end
end end
Timecop.travel(job.updated_at + update_interval) do Timecop.travel(job.updated_at + update_interval) do
......
...@@ -330,7 +330,7 @@ describe API::V3::Builds, api: true do ...@@ -330,7 +330,7 @@ describe API::V3::Builds, api: true do
context 'authorized user' do context 'authorized user' do
it 'returns specific job trace' do it 'returns specific job trace' do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(response.body).to eq(build.trace) expect(response.body).to eq(build.trace.raw)
end end
end end
...@@ -418,7 +418,7 @@ describe API::V3::Builds, api: true do ...@@ -418,7 +418,7 @@ describe API::V3::Builds, api: true do
it 'erases job content' do it 'erases job content' do
expect(response.status).to eq 201 expect(response.status).to eq 201
expect(build.trace).to be_empty expect(build).not_to have_trace
expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_file.exists?).to be_falsy
expect(build.artifacts_metadata.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy
end end
......
...@@ -285,7 +285,7 @@ describe Ci::API::Builds do ...@@ -285,7 +285,7 @@ describe Ci::API::Builds do
end end
it 'does not override trace information when no trace is given' do it 'does not override trace information when no trace is given' do
expect(build.reload.trace).to eq 'BUILD TRACE' expect(build.reload.trace.raw).to eq 'BUILD TRACE'
end end
context 'job has been erased' do context 'job has been erased' do
...@@ -309,9 +309,11 @@ describe Ci::API::Builds do ...@@ -309,9 +309,11 @@ describe Ci::API::Builds do
def patch_the_trace(content = ' appended', request_headers = nil) def patch_the_trace(content = ' appended', request_headers = nil)
unless request_headers unless request_headers
offset = build.trace_length build.trace.read do |stream|
limit = offset + content.length - 1 offset = stream.size
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) limit = offset + content.length - 1
request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" })
end
end end
Timecop.travel(build.updated_at + update_interval) do Timecop.travel(build.updated_at + update_interval) do
...@@ -335,7 +337,7 @@ describe Ci::API::Builds do ...@@ -335,7 +337,7 @@ describe Ci::API::Builds do
context 'when request is valid' do context 'when request is valid' do
it 'gets correct response' do it 'gets correct response' do
expect(response.status).to eq 202 expect(response.status).to eq 202
expect(build.reload.trace).to eq 'BUILD TRACE appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Build-Status' expect(response.header).to have_key 'Build-Status'
end end
...@@ -346,7 +348,7 @@ describe Ci::API::Builds do ...@@ -346,7 +348,7 @@ describe Ci::API::Builds do
it 'changes the build trace' do it 'changes the build trace' do
patch_the_trace patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end end
context 'when Runner makes a force-patch' do context 'when Runner makes a force-patch' do
...@@ -355,7 +357,7 @@ describe Ci::API::Builds do ...@@ -355,7 +357,7 @@ describe Ci::API::Builds do
it "doesn't change the build.trace" do it "doesn't change the build.trace" do
force_patch_the_trace force_patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
end end
end end
end end
...@@ -368,7 +370,7 @@ describe Ci::API::Builds do ...@@ -368,7 +370,7 @@ describe Ci::API::Builds do
it 'changes the build.trace' do it 'changes the build.trace' do
patch_the_trace patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended'
end end
context 'when Runner makes a force-patch' do context 'when Runner makes a force-patch' do
...@@ -377,7 +379,7 @@ describe Ci::API::Builds do ...@@ -377,7 +379,7 @@ describe Ci::API::Builds do
it "doesn't change the build.trace" do it "doesn't change the build.trace" do
force_patch_the_trace force_patch_the_trace
expect(build.reload.trace).to eq 'BUILD TRACE appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
end end
end end
end end
...@@ -403,7 +405,7 @@ describe Ci::API::Builds do ...@@ -403,7 +405,7 @@ describe Ci::API::Builds do
it 'gets correct response' do it 'gets correct response' do
expect(response.status).to eq 202 expect(response.status).to eq 202
expect(build.reload.trace).to eq 'BUILD TRACE appended' expect(build.reload.trace.raw).to eq 'BUILD TRACE appended'
expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Range'
expect(response.header).to have_key 'Build-Status' expect(response.header).to have_key 'Build-Status'
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment