Commit 5c168885 authored by Fabio Pitino's avatar Fabio Pitino Committed by Rémy Coutable

Expose arbitrary artifacts via MR widget

* Allow user to specify `artifacts:expose_as` in CI config
* Save :has_exposed_artifacts in job metadata for queries
* Find exposed artifacts in build metadata model
* Expose API endpoint for frontend to fetch data

Fix unlrelated controller specs

Use default has_exposed_artifacts NULL

Avoid using a background migration to change NULL
to false. It's not needed.

Feedback from review

* add links to issue for follow up refactoring
* preload job artifacts and metadata associations
* merge DisallowedRegexInArrayValidator into existing
ArrayOfStringsValidator
* other minor changes

Rename params to match frontend code

Feedback from review

Feedback from review
parent fae81201
...@@ -13,7 +13,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo ...@@ -13,7 +13,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
skip_before_action :merge_request, only: [:index, :bulk_update] skip_before_action :merge_request, only: [:index, :bulk_update]
before_action :whitelist_query_limiting, only: [:assign_related_issues, :update] before_action :whitelist_query_limiting, only: [:assign_related_issues, :update]
before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort] before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort]
before_action :authorize_test_reports!, only: [:test_reports] before_action :authorize_read_actual_head_pipeline!, only: [:test_reports, :exposed_artifacts]
before_action :set_issuables_index, only: [:index] before_action :set_issuables_index, only: [:index]
before_action :authenticate_user!, only: [:assign_related_issues] before_action :authenticate_user!, only: [:assign_related_issues]
before_action :check_user_can_push_to_source_branch!, only: [:rebase] before_action :check_user_can_push_to_source_branch!, only: [:rebase]
...@@ -115,6 +115,14 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo ...@@ -115,6 +115,14 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
reports_response(@merge_request.compare_test_reports) reports_response(@merge_request.compare_test_reports)
end end
def exposed_artifacts
if @merge_request.has_exposed_artifacts?
reports_response(@merge_request.find_exposed_artifacts)
else
head :no_content
end
end
def edit def edit
define_edit_vars define_edit_vars
end end
...@@ -357,8 +365,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo ...@@ -357,8 +365,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
end end
end end
def authorize_test_reports! def authorize_read_actual_head_pipeline!
# MergeRequest#actual_head_pipeline is the pipeline accessed in MergeRequest#compare_reports.
return render_404 unless can?(current_user, :read_build, merge_request.actual_head_pipeline) return render_404 unless can?(current_user, :read_build, merge_request.actual_head_pipeline)
end end
end end
......
...@@ -118,6 +118,11 @@ module Ci ...@@ -118,6 +118,11 @@ module Ci
scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
scope :with_exposed_artifacts, -> do
joins(:metadata).merge(Ci::BuildMetadata.with_exposed_artifacts)
.includes(:metadata, :job_artifacts_metadata)
end
scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
...@@ -595,6 +600,14 @@ module Ci ...@@ -595,6 +600,14 @@ module Ci
update_column(:trace, nil) update_column(:trace, nil)
end end
def artifacts_expose_as
options.dig(:artifacts, :expose_as)
end
def artifacts_paths
options.dig(:artifacts, :paths)
end
def needs_touch? def needs_touch?
Time.now - updated_at > 15.minutes.to_i Time.now - updated_at > 15.minutes.to_i
end end
......
...@@ -27,6 +27,7 @@ module Ci ...@@ -27,6 +27,7 @@ module Ci
scope :scoped_build, -> { where('ci_builds_metadata.build_id = ci_builds.id') } scope :scoped_build, -> { where('ci_builds_metadata.build_id = ci_builds.id') }
scope :with_interruptible, -> { where(interruptible: true) } scope :with_interruptible, -> { where(interruptible: true) }
scope :with_exposed_artifacts, -> { where(has_exposed_artifacts: true) }
enum timeout_source: { enum timeout_source: {
unknown_timeout_source: 1, unknown_timeout_source: 1,
......
...@@ -783,6 +783,10 @@ module Ci ...@@ -783,6 +783,10 @@ module Ci
end end
end end
def has_exposed_artifacts?
complete? && builds.latest.with_exposed_artifacts.exists?
end
def branch_updated? def branch_updated?
strong_memoize(:branch_updated) do strong_memoize(:branch_updated) do
push_details.branch_updated? push_details.branch_updated?
......
...@@ -16,6 +16,7 @@ module Ci ...@@ -16,6 +16,7 @@ module Ci
delegate :timeout, to: :metadata, prefix: true, allow_nil: true delegate :timeout, to: :metadata, prefix: true, allow_nil: true
delegate :interruptible, to: :metadata, prefix: false, allow_nil: true delegate :interruptible, to: :metadata, prefix: false, allow_nil: true
delegate :has_exposed_artifacts?, to: :metadata, prefix: false, allow_nil: true
before_create :ensure_metadata before_create :ensure_metadata
end end
...@@ -45,6 +46,9 @@ module Ci ...@@ -45,6 +46,9 @@ module Ci
def options=(value) def options=(value)
write_metadata_attribute(:options, :config_options, value) write_metadata_attribute(:options, :config_options, value)
# Store presence of exposed artifacts in build metadata to make it easier to query
ensure_metadata.has_exposed_artifacts = value&.dig(:artifacts, :expose_as).present?
end end
def yaml_variables=(value) def yaml_variables=(value)
......
...@@ -1255,6 +1255,27 @@ class MergeRequest < ApplicationRecord ...@@ -1255,6 +1255,27 @@ class MergeRequest < ApplicationRecord
compare_reports(Ci::CompareTestReportsService) compare_reports(Ci::CompareTestReportsService)
end end
def has_exposed_artifacts?
return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
actual_head_pipeline&.has_exposed_artifacts?
end
# TODO: this method and compare_test_reports use the same
# result type, which is handled by the controller's #reports_response.
# we should minimize mistakes by isolating the common parts.
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
def find_exposed_artifacts
unless has_exposed_artifacts?
return { status: :error, status_reason: 'This merge request does not have exposed artifacts' }
end
compare_reports(Ci::GenerateExposedArtifactsReportService)
end
# TODO: consider renaming this as with exposed artifacts we generate reports,
# not always compare
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
def compare_reports(service_class, current_user = nil) def compare_reports(service_class, current_user = nil)
with_reactive_cache(service_class.name, current_user&.id) do |data| with_reactive_cache(service_class.name, current_user&.id) do |data|
unless service_class.new(project, current_user) unless service_class.new(project, current_user)
...@@ -1269,6 +1290,8 @@ class MergeRequest < ApplicationRecord ...@@ -1269,6 +1290,8 @@ class MergeRequest < ApplicationRecord
def calculate_reactive_cache(identifier, current_user_id = nil, *args) def calculate_reactive_cache(identifier, current_user_id = nil, *args)
service_class = identifier.constantize service_class = identifier.constantize
# TODO: the type check should change to something that includes exposed artifacts service
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
raise NameError, service_class unless service_class < Ci::CompareReportsBaseService raise NameError, service_class unless service_class < Ci::CompareReportsBaseService
current_user = User.find_by(id: current_user_id) current_user = User.find_by(id: current_user_id)
......
...@@ -65,6 +65,12 @@ class MergeRequestPollWidgetEntity < IssuableEntity ...@@ -65,6 +65,12 @@ class MergeRequestPollWidgetEntity < IssuableEntity
end end
end end
expose :exposed_artifacts_path do |merge_request|
if merge_request.has_exposed_artifacts?
exposed_artifacts_project_merge_request_path(merge_request.project, merge_request, format: :json)
end
end
expose :create_issue_to_resolve_discussions_path do |merge_request| expose :create_issue_to_resolve_discussions_path do |merge_request|
presenter(merge_request).create_issue_to_resolve_discussions_path presenter(merge_request).create_issue_to_resolve_discussions_path
end end
......
# frozen_string_literal: true # frozen_string_literal: true
module Ci module Ci
# TODO: when using this class with exposed artifacts we see that there are
# 2 responsibilities:
# 1. reactive caching interface (same in all cases)
# 2. data generator (report comparison in most of the case but not always)
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
class CompareReportsBaseService < ::BaseService class CompareReportsBaseService < ::BaseService
def execute(base_pipeline, head_pipeline) def execute(base_pipeline, head_pipeline)
comparer = comparer_class.new(get_report(base_pipeline), get_report(head_pipeline)) comparer = comparer_class.new(get_report(base_pipeline), get_report(head_pipeline))
......
# frozen_string_literal: true
module Ci
# This class loops through all builds with exposed artifacts and returns
# basic information about exposed artifacts for given jobs for the frontend
# to display them as custom links in the merge request.
#
# This service must be used with care.
# Looking for exposed artifacts is very slow and should be done asynchronously.
class FindExposedArtifactsService < ::BaseService
include Gitlab::Routing
MAX_EXPOSED_ARTIFACTS = 10
def for_pipeline(pipeline, limit: MAX_EXPOSED_ARTIFACTS)
results = []
pipeline.builds.latest.with_exposed_artifacts.find_each do |job|
if job_exposed_artifacts = for_job(job)
results << job_exposed_artifacts
end
break if results.size >= limit
end
results
end
def for_job(job)
return unless job.has_exposed_artifacts?
metadata_entries = first_2_metadata_entries_for_artifacts_paths(job)
return if metadata_entries.empty?
{
text: job.artifacts_expose_as,
url: path_for_entries(metadata_entries, job),
job_path: project_job_path(project, job),
job_name: job.name
}
end
private
# we don't need to fetch all artifacts entries for a job because
# it could contain many. We only need to know whether it has 1 or more
# artifacts, so fetching the first 2 would be sufficient.
def first_2_metadata_entries_for_artifacts_paths(job)
job.artifacts_paths
.lazy
.map { |path| job.artifacts_metadata_entry(path, recursive: true) }
.select { |entry| entry.exists? }
.first(2)
end
def path_for_entries(entries, job)
return if entries.empty?
if single_artifact?(entries)
file_project_job_artifacts_path(project, job, entries.first.path)
else
browse_project_job_artifacts_path(project, job)
end
end
def single_artifact?(entries)
entries.size == 1 && entries.first.file?
end
end
end
# frozen_string_literal: true
module Ci
# TODO: a couple of points with this approach:
# + reuses existing architecture and reactive caching
# - it's not a report comparison and some comparing features must be turned off.
# see CompareReportsBaseService for more notes.
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
class GenerateExposedArtifactsReportService < CompareReportsBaseService
def execute(base_pipeline, head_pipeline)
data = FindExposedArtifactsService.new(project, current_user).for_pipeline(head_pipeline)
{
status: :parsed,
key: key(base_pipeline, head_pipeline),
data: data
}
rescue => e
Gitlab::Sentry.track_acceptable_exception(e, extra: { project_id: project.id })
{
status: :error,
key: key(base_pipeline, head_pipeline),
status_reason: _('An error occurred while fetching exposed artifacts.')
}
end
def latest?(base_pipeline, head_pipeline, data)
data&.fetch(:key, nil) == key(base_pipeline, head_pipeline)
end
end
end
---
title: Expose arbitrary job artifacts in Merge Request widget
merge_request: 18385
author:
type: added
...@@ -274,6 +274,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do ...@@ -274,6 +274,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
get :discussions, format: :json get :discussions, format: :json
post :rebase post :rebase
get :test_reports get :test_reports
get :exposed_artifacts
scope constraints: { format: nil }, action: :show do scope constraints: { format: nil }, action: :show do
get :commits, defaults: { tab: 'commits' } get :commits, defaults: { tab: 'commits' }
......
# frozen_string_literal: true
class AddHasExposedArtifactsToCiBuildsMetadata < ActiveRecord::Migration[5.2]
DOWNTIME = false
def up
add_column :ci_builds_metadata, :has_exposed_artifacts, :boolean
end
def down
remove_column :ci_builds_metadata, :has_exposed_artifacts
end
end
# frozen_string_literal: true
class AddIndexToCiBuildsMetadataHasExposedArtifacts < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_builds_metadata, [:build_id], where: "has_exposed_artifacts IS TRUE", name: 'index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts'
end
def down
remove_concurrent_index_by_name :ci_builds_metadata, 'index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts'
end
end
...@@ -691,7 +691,9 @@ ActiveRecord::Schema.define(version: 2019_10_16_220135) do ...@@ -691,7 +691,9 @@ ActiveRecord::Schema.define(version: 2019_10_16_220135) do
t.boolean "interruptible" t.boolean "interruptible"
t.jsonb "config_options" t.jsonb "config_options"
t.jsonb "config_variables" t.jsonb "config_variables"
t.boolean "has_exposed_artifacts"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts", where: "(has_exposed_artifacts IS TRUE)"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id_and_interruptible", where: "(interruptible = true)" t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id_and_interruptible", where: "(interruptible = true)"
t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id" t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id"
end end
......
...@@ -12,7 +12,9 @@ module Gitlab ...@@ -12,7 +12,9 @@ module Gitlab
include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Attributable
ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze
EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze
EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces"
attributes ALLOWED_KEYS attributes ALLOWED_KEYS
...@@ -21,11 +23,18 @@ module Gitlab ...@@ -21,11 +23,18 @@ module Gitlab
validations do validations do
validates :config, type: Hash validates :config, type: Hash
validates :config, allowed_keys: ALLOWED_KEYS validates :config, allowed_keys: ALLOWED_KEYS
validates :paths, presence: true, if: :expose_as_present?
with_options allow_nil: true do with_options allow_nil: true do
validates :name, type: String validates :name, type: String
validates :untracked, boolean: true validates :untracked, boolean: true
validates :paths, array_of_strings: true validates :paths, array_of_strings: true
validates :paths, array_of_strings: {
with: /\A[^*]*\z/,
message: "can't contain '*' when used with 'expose_as'"
}, if: :expose_as_present?
validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present?
validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present?
validates :reports, type: Hash validates :reports, type: Hash
validates :when, validates :when,
inclusion: { in: %w[on_success on_failure always], inclusion: { in: %w[on_success on_failure always],
...@@ -41,6 +50,12 @@ module Gitlab ...@@ -41,6 +50,12 @@ module Gitlab
@config[:reports] = reports_value if @config.key?(:reports) @config[:reports] = reports_value if @config.key?(:reports)
@config @config
end end
def expose_as_present?
return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
!@config[:expose_as].nil?
end
end end
end end
end end
......
...@@ -61,8 +61,15 @@ module Gitlab ...@@ -61,8 +61,15 @@ module Gitlab
include LegacyValidationHelpers include LegacyValidationHelpers
def validate_each(record, attribute, value) def validate_each(record, attribute, value)
unless validate_array_of_strings(value) valid = validate_array_of_strings(value)
record.errors.add(attribute, 'should be an array of strings')
record.errors.add(attribute, 'should be an array of strings') unless valid
if valid && options[:with]
unless value.all? { |v| v =~ options[:with] }
message = options[:message] || 'contains elements that do not match the format'
record.errors.add(attribute, message)
end
end end
end end
end end
......
...@@ -1515,6 +1515,9 @@ msgstr "" ...@@ -1515,6 +1515,9 @@ msgstr ""
msgid "An error occurred while fetching environments." msgid "An error occurred while fetching environments."
msgstr "" msgstr ""
msgid "An error occurred while fetching exposed artifacts."
msgstr ""
msgid "An error occurred while fetching folder content." msgid "An error occurred while fetching folder content."
msgstr "" msgstr ""
......
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
describe Projects::MergeRequestsController do describe Projects::MergeRequestsController do
include ProjectForksHelper include ProjectForksHelper
include Gitlab::Routing
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:user) { project.owner } let(:user) { project.owner }
...@@ -206,7 +207,7 @@ describe Projects::MergeRequestsController do ...@@ -206,7 +207,7 @@ describe Projects::MergeRequestsController do
it 'redirects to last_page if page number is larger than number of pages' do it 'redirects to last_page if page number is larger than number of pages' do
get_merge_requests(last_page + 1) get_merge_requests(last_page + 1)
expect(response).to redirect_to(namespace_project_merge_requests_path(page: last_page, state: controller.params[:state], scope: controller.params[:scope])) expect(response).to redirect_to(project_merge_requests_path(project, page: last_page, state: controller.params[:state], scope: controller.params[:scope]))
end end
it 'redirects to specified page' do it 'redirects to specified page' do
...@@ -227,7 +228,7 @@ describe Projects::MergeRequestsController do ...@@ -227,7 +228,7 @@ describe Projects::MergeRequestsController do
host: external_host host: external_host
} }
expect(response).to redirect_to(namespace_project_merge_requests_path(page: last_page, state: controller.params[:state], scope: controller.params[:scope])) expect(response).to redirect_to(project_merge_requests_path(project, page: last_page, state: controller.params[:state], scope: controller.params[:scope]))
end end
end end
...@@ -770,6 +771,189 @@ describe Projects::MergeRequestsController do ...@@ -770,6 +771,189 @@ describe Projects::MergeRequestsController do
end end
end end
describe 'GET exposed_artifacts' do
let(:merge_request) do
create(:merge_request,
:with_merge_request_pipeline,
target_project: project,
source_project: project)
end
let(:pipeline) do
create(:ci_pipeline,
:success,
project: merge_request.source_project,
ref: merge_request.source_branch,
sha: merge_request.diff_head_sha)
end
let!(:job) { create(:ci_build, pipeline: pipeline, options: job_options) }
let!(:job_metadata) { create(:ci_job_artifact, :metadata, job: job) }
before do
allow_any_instance_of(MergeRequest)
.to receive(:find_exposed_artifacts)
.and_return(report)
allow_any_instance_of(MergeRequest)
.to receive(:actual_head_pipeline)
.and_return(pipeline)
end
subject do
get :exposed_artifacts, params: {
namespace_id: project.namespace.to_param,
project_id: project,
id: merge_request.iid
},
format: :json
end
describe 'permissions on a public project with private CI/CD' do
let(:project) { create :project, :repository, :public, :builds_private }
let(:report) { { status: :parsed, data: [] } }
let(:job_options) { {} }
context 'while signed out' do
before do
sign_out(user)
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(404)
expect(response.body).to be_blank
end
end
context 'while signed in as an unrelated user' do
before do
sign_in(create(:user))
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(404)
expect(response.body).to be_blank
end
end
end
context 'when pipeline has jobs with exposed artifacts' do
let(:job_options) do
{
artifacts: {
paths: ['ci_artifacts.txt'],
expose_as: 'Exposed artifact'
}
}
end
context 'when fetching exposed artifacts is in progress' do
let(:report) { { status: :parsing } }
it 'sends polling interval' do
expect(Gitlab::PollingInterval).to receive(:set_header)
subject
end
it 'returns 204 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'when fetching exposed artifacts is completed' do
let(:data) do
Ci::GenerateExposedArtifactsReportService.new(project, user)
.execute(nil, pipeline)
end
let(:report) { { status: :parsed, data: data } }
it 'returns exposed artifacts' do
subject
expect(response).to have_gitlab_http_status(200)
expect(json_response['status']).to eq('parsed')
expect(json_response['data']).to eq([{
'job_name' => 'test',
'job_path' => project_job_path(project, job),
'url' => file_project_job_artifacts_path(project, job, 'ci_artifacts.txt'),
'text' => 'Exposed artifact'
}])
end
end
context 'when something went wrong on our system' do
let(:report) { {} }
it 'does not send polling interval' do
expect(Gitlab::PollingInterval).not_to receive(:set_header)
subject
end
it 'returns 500 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:internal_server_error)
expect(json_response).to eq({ 'status_reason' => 'Unknown error' })
end
end
context 'when feature flag :ci_expose_arbitrary_artifacts_in_mr is disabled' do
let(:job_options) do
{
artifacts: {
paths: ['ci_artifacts.txt'],
expose_as: 'Exposed artifact'
}
}
end
let(:report) { double }
before do
stub_feature_flags(ci_expose_arbitrary_artifacts_in_mr: false)
end
it 'does not send polling interval' do
expect(Gitlab::PollingInterval).not_to receive(:set_header)
subject
end
it 'returns 204 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:no_content)
end
end
end
context 'when pipeline does not have jobs with exposed artifacts' do
let(:report) { double }
let(:job_options) do
{
artifacts: {
paths: ['ci_artifacts.txt']
}
}
end
it 'returns no content' do
subject
expect(response).to have_gitlab_http_status(204)
expect(response.body).to be_empty
end
end
end
describe 'GET test_reports' do describe 'GET test_reports' do
let(:merge_request) do let(:merge_request) do
create(:merge_request, create(:merge_request,
......
...@@ -95,6 +95,17 @@ FactoryBot.define do ...@@ -95,6 +95,17 @@ FactoryBot.define do
end end
end end
trait :with_exposed_artifacts do
status { :success }
after(:build) do |pipeline, evaluator|
pipeline.builds << build(:ci_build, :artifacts,
pipeline: pipeline,
project: pipeline.project,
options: { artifacts: { expose_as: 'the artifact', paths: ['ci_artifacts.txt'] } })
end
end
trait :with_job do trait :with_job do
after(:build) do |pipeline, evaluator| after(:build) do |pipeline, evaluator|
pipeline.builds << build(:ci_build, pipeline: pipeline, project: pipeline.project) pipeline.builds << build(:ci_build, pipeline: pipeline, project: pipeline.project)
......
...@@ -120,6 +120,18 @@ FactoryBot.define do ...@@ -120,6 +120,18 @@ FactoryBot.define do
end end
end end
trait :with_exposed_artifacts do
after(:build) do |merge_request|
merge_request.head_pipeline = build(
:ci_pipeline,
:success,
:with_exposed_artifacts,
project: merge_request.source_project,
ref: merge_request.source_branch,
sha: merge_request.diff_head_sha)
end
end
trait :with_legacy_detached_merge_request_pipeline do trait :with_legacy_detached_merge_request_pipeline do
after(:create) do |merge_request| after(:create) do |merge_request|
merge_request.pipelines_for_merge_request << create(:ci_pipeline, merge_request.pipelines_for_merge_request << create(:ci_pipeline,
......
...@@ -5,6 +5,7 @@ require 'spec_helper' ...@@ -5,6 +5,7 @@ require 'spec_helper'
describe 'Merge request > User sees merge widget', :js do describe 'Merge request > User sees merge widget', :js do
include ProjectForksHelper include ProjectForksHelper
include TestReportsHelper include TestReportsHelper
include ReactiveCachingHelpers
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:project_only_mwps) { create(:project, :repository, only_allow_merge_if_pipeline_succeeds: true) } let(:project_only_mwps) { create(:project, :repository, only_allow_merge_if_pipeline_succeeds: true) }
...@@ -435,6 +436,54 @@ describe 'Merge request > User sees merge widget', :js do ...@@ -435,6 +436,54 @@ describe 'Merge request > User sees merge widget', :js do
end end
end end
context 'exposed artifacts' do
subject { visit project_merge_request_path(project, merge_request) }
context 'when merge request has exposed artifacts' do
let(:merge_request) { create(:merge_request, :with_exposed_artifacts, source_project: project) }
let(:job) { merge_request.head_pipeline.builds.last }
let!(:artifacts_metadata) { create(:ci_job_artifact, :metadata, job: job) }
context 'when result has not been parsed yet' do
it 'shows parsing status' do
subject
expect(page).to have_content('Loading artifacts')
end
end
context 'when result has been parsed' do
before do
allow_any_instance_of(MergeRequest).to receive(:find_exposed_artifacts).and_return(
status: :parsed, data: [
{
text: "the artifact",
url: "/namespace1/project1/-/jobs/1/artifacts/file/ci_artifacts.txt",
job_path: "/namespace1/project1/-/jobs/1",
job_name: "test"
}
])
end
it 'shows the parsed results' do
subject
expect(page).to have_content('View exposed artifact')
end
end
end
context 'when merge request does not have exposed artifacts' do
let(:merge_request) { create(:merge_request, source_project: project) }
it 'does not show parsing status' do
subject
expect(page).not_to have_content('Loading artifacts')
end
end
end
context 'when merge request has test reports' do context 'when merge request has test reports' do
let!(:head_pipeline) do let!(:head_pipeline) do
create(:ci_pipeline, create(:ci_pipeline,
......
...@@ -28,6 +28,14 @@ describe Gitlab::Ci::Config::Entry::Artifacts do ...@@ -28,6 +28,14 @@ describe Gitlab::Ci::Config::Entry::Artifacts do
expect(entry.value).to eq config expect(entry.value).to eq config
end end
end end
context "when value includes 'expose_as' keyword" do
let(:config) { { paths: %w[results.txt], expose_as: "Test results" } }
it 'returns general artifact and report-type artifacts configuration' do
expect(entry.value).to eq config
end
end
end end
context 'when entry value is not correct' do context 'when entry value is not correct' do
...@@ -58,6 +66,84 @@ describe Gitlab::Ci::Config::Entry::Artifacts do ...@@ -58,6 +66,84 @@ describe Gitlab::Ci::Config::Entry::Artifacts do
.to include 'artifacts reports should be a hash' .to include 'artifacts reports should be a hash'
end end
end end
context "when 'expose_as' is not a string" do
let(:config) { { paths: %w[results.txt], expose_as: 1 } }
it 'reports error' do
expect(entry.errors)
.to include 'artifacts expose as should be a string'
end
end
context "when 'expose_as' is too long" do
let(:config) { { paths: %w[results.txt], expose_as: 'A' * 101 } }
it 'reports error' do
expect(entry.errors)
.to include 'artifacts expose as is too long (maximum is 100 characters)'
end
end
context "when 'expose_as' is an empty string" do
let(:config) { { paths: %w[results.txt], expose_as: '' } }
it 'reports error' do
expect(entry.errors)
.to include 'artifacts expose as ' + Gitlab::Ci::Config::Entry::Artifacts::EXPOSE_AS_ERROR_MESSAGE
end
end
context "when 'expose_as' contains invalid characters" do
let(:config) do
{ paths: %w[results.txt], expose_as: '<script>alert("xss");</script>' }
end
it 'reports error' do
expect(entry.errors)
.to include 'artifacts expose as ' + Gitlab::Ci::Config::Entry::Artifacts::EXPOSE_AS_ERROR_MESSAGE
end
end
context "when 'expose_as' is used without 'paths'" do
let(:config) { { expose_as: 'Test results' } }
it 'reports error' do
expect(entry.errors)
.to include "artifacts paths can't be blank"
end
end
context "when 'paths' includes '*' and 'expose_as' is defined" do
let(:config) { { expose_as: 'Test results', paths: ['test.txt', 'test*.txt'] } }
it 'reports error' do
expect(entry.errors)
.to include "artifacts paths can't contain '*' when used with 'expose_as'"
end
end
end
context 'when feature flag :ci_expose_arbitrary_artifacts_in_mr is disabled' do
before do
stub_feature_flags(ci_expose_arbitrary_artifacts_in_mr: false)
end
context 'when syntax is correct' do
let(:config) { { expose_as: 'Test results', paths: ['test.txt'] } }
it 'is valid' do
expect(entry.errors).to be_empty
end
end
context 'when syntax for :expose_as is incorrect' do
let(:config) { { paths: %w[results.txt], expose_as: '' } }
it 'is valid' do
expect(entry.errors).to be_empty
end
end
end end
end end
end end
......
...@@ -970,6 +970,7 @@ module Gitlab ...@@ -970,6 +970,7 @@ module Gitlab
rspec: { rspec: {
artifacts: { artifacts: {
paths: ["logs/", "binaries/"], paths: ["logs/", "binaries/"],
expose_as: "Exposed artifacts",
untracked: true, untracked: true,
name: "custom_name", name: "custom_name",
expire_in: "7d" expire_in: "7d"
...@@ -993,6 +994,7 @@ module Gitlab ...@@ -993,6 +994,7 @@ module Gitlab
artifacts: { artifacts: {
name: "custom_name", name: "custom_name",
paths: ["logs/", "binaries/"], paths: ["logs/", "binaries/"],
expose_as: "Exposed artifacts",
untracked: true, untracked: true,
expire_in: "7d" expire_in: "7d"
} }
......
...@@ -206,6 +206,35 @@ describe Ci::Build do ...@@ -206,6 +206,35 @@ describe Ci::Build do
end end
end end
describe '.with_exposed_artifacts' do
subject { described_class.with_exposed_artifacts }
let!(:job1) { create(:ci_build) }
let!(:job2) { create(:ci_build, options: options) }
let!(:job3) { create(:ci_build) }
context 'when some jobs have exposed artifacs and some not' do
let(:options) { { artifacts: { expose_as: 'test', paths: ['test'] } } }
before do
job1.ensure_metadata.update!(has_exposed_artifacts: nil)
job3.ensure_metadata.update!(has_exposed_artifacts: false)
end
it 'selects only the jobs with exposed artifacts' do
is_expected.to eq([job2])
end
end
context 'when job does not expose artifacts' do
let(:options) { nil }
it 'returns an empty array' do
is_expected.to be_empty
end
end
end
describe '.with_reports' do describe '.with_reports' do
subject { described_class.with_reports(Ci::JobArtifact.test_reports) } subject { described_class.with_reports(Ci::JobArtifact.test_reports) }
...@@ -1844,6 +1873,14 @@ describe Ci::Build do ...@@ -1844,6 +1873,14 @@ describe Ci::Build do
expect(build.metadata.read_attribute(:config_options)).to be_nil expect(build.metadata.read_attribute(:config_options)).to be_nil
end end
end end
context 'when options include artifacts:expose_as' do
let(:build) { create(:ci_build, options: { artifacts: { expose_as: 'test' } }) }
it 'saves the presence of expose_as into build metadata' do
expect(build.metadata).to have_exposed_artifacts
end
end
end end
describe '#other_manual_actions' do describe '#other_manual_actions' do
......
...@@ -1674,6 +1674,63 @@ describe MergeRequest do ...@@ -1674,6 +1674,63 @@ describe MergeRequest do
end end
end end
describe '#find_exposed_artifacts' do
let(:project) { create(:project, :repository) }
let(:merge_request) { create(:merge_request, :with_test_reports, source_project: project) }
let(:pipeline) { merge_request.head_pipeline }
subject { merge_request.find_exposed_artifacts }
context 'when head pipeline has exposed artifacts' do
let!(:job) do
create(:ci_build, options: { artifacts: { expose_as: 'artifact', paths: ['ci_artifacts.txt'] } }, pipeline: pipeline)
end
let!(:artifacts_metadata) { create(:ci_job_artifact, :metadata, job: job) }
context 'when reactive cache worker is parsing results asynchronously' do
it 'returns status' do
expect(subject[:status]).to eq(:parsing)
end
end
context 'when reactive cache worker is inline' do
before do
synchronous_reactive_cache(merge_request)
end
it 'returns status and data' do
expect(subject[:status]).to eq(:parsed)
end
context 'when an error occurrs' do
before do
expect_next_instance_of(Ci::FindExposedArtifactsService) do |service|
expect(service).to receive(:for_pipeline)
.and_raise(StandardError.new)
end
end
it 'returns an error message' do
expect(subject[:status]).to eq(:error)
end
end
context 'when cached results is not latest' do
before do
allow_next_instance_of(Ci::GenerateExposedArtifactsReportService) do |service|
allow(service).to receive(:latest?).and_return(false)
end
end
it 'raises and InvalidateReactiveCache error' do
expect { subject }.to raise_error(ReactiveCaching::InvalidateReactiveCache)
end
end
end
end
end
describe '#compare_test_reports' do describe '#compare_test_reports' do
subject { merge_request.compare_test_reports } subject { merge_request.compare_test_reports }
......
...@@ -358,4 +358,26 @@ describe MergeRequestWidgetEntity do ...@@ -358,4 +358,26 @@ describe MergeRequestWidgetEntity do
end end
end end
end end
describe 'exposed_artifacts_path' do
context 'when merge request has exposed artifacts' do
before do
expect(resource).to receive(:has_exposed_artifacts?).and_return(true)
end
it 'set the path to poll data' do
expect(subject[:exposed_artifacts_path]).to be_present
end
end
context 'when merge request has no exposed artifacts' do
before do
expect(resource).to receive(:has_exposed_artifacts?).and_return(false)
end
it 'set the path to poll data' do
expect(subject[:exposed_artifacts_path]).to be_nil
end
end
end
end end
# frozen_string_literal: true
require 'spec_helper'
describe Ci::FindExposedArtifactsService do
include Gitlab::Routing
let(:metadata) do
Gitlab::Ci::Build::Artifacts::Metadata
.new(metadata_file_stream, path, { recursive: true })
end
let(:metadata_file_stream) do
File.open(Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz')
end
let_it_be(:project) { create(:project) }
let(:user) { nil }
after do
metadata_file_stream&.close
end
def create_job_with_artifacts(options)
create(:ci_build, pipeline: pipeline, options: options).tap do |job|
create(:ci_job_artifact, :metadata, job: job)
end
end
describe '#for_pipeline' do
shared_examples 'finds a single match' do
it 'returns the artifact with exact location' do
expect(subject).to eq([{
text: 'Exposed artifact',
url: file_project_job_artifacts_path(project, job, 'other_artifacts_0.1.2/doc_sample.txt'),
job_name: job.name,
job_path: project_job_path(project, job)
}])
end
end
shared_examples 'finds multiple matches' do
it 'returns the path to the artifacts browser' do
expect(subject).to eq([{
text: 'Exposed artifact',
url: browse_project_job_artifacts_path(project, job),
job_name: job.name,
job_path: project_job_path(project, job)
}])
end
end
let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
subject { described_class.new(project, user).for_pipeline(pipeline) }
context 'with jobs having at most 1 matching exposed artifact' do
let!(:job) do
create_job_with_artifacts(artifacts: {
expose_as: 'Exposed artifact',
paths: ['other_artifacts_0.1.2/doc_sample.txt', 'something-else.html']
})
end
it_behaves_like 'finds a single match'
end
context 'with jobs having more than 1 matching exposed artifacts' do
let!(:job) do
create_job_with_artifacts(artifacts: {
expose_as: 'Exposed artifact',
paths: [
'ci_artifacts.txt',
'other_artifacts_0.1.2/doc_sample.txt',
'something-else.html'
]
})
end
it_behaves_like 'finds multiple matches'
end
context 'with jobs having more than 1 matching exposed artifacts inside a directory' do
let!(:job) do
create_job_with_artifacts(artifacts: {
expose_as: 'Exposed artifact',
paths: ['tests_encoding/']
})
end
it_behaves_like 'finds multiple matches'
end
context 'with jobs having paths with glob expression' do
let!(:job) do
create_job_with_artifacts(artifacts: {
expose_as: 'Exposed artifact',
paths: ['other_artifacts_0.1.2/doc_sample.txt', 'tests_encoding/*.*']
})
end
it_behaves_like 'finds a single match' # because those with * are ignored
end
context 'limiting results' do
let!(:job1) do
create_job_with_artifacts(artifacts: {
expose_as: 'artifact 1',
paths: ['ci_artifacts.txt']
})
end
let!(:job2) do
create_job_with_artifacts(artifacts: {
expose_as: 'artifact 2',
paths: ['tests_encoding/']
})
end
let!(:job3) do
create_job_with_artifacts(artifacts: {
expose_as: 'should not be exposed',
paths: ['other_artifacts_0.1.2/doc_sample.txt']
})
end
subject { described_class.new(project, user).for_pipeline(pipeline, limit: 2) }
it 'returns first 2 results' do
expect(subject).to eq([
{
text: 'artifact 1',
url: file_project_job_artifacts_path(project, job1, 'ci_artifacts.txt'),
job_name: job1.name,
job_path: project_job_path(project, job1)
},
{
text: 'artifact 2',
url: browse_project_job_artifacts_path(project, job2),
job_name: job2.name,
job_path: project_job_path(project, job2)
}
])
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment