Commit c4d615c9 authored by Kamil Trzciński's avatar Kamil Trzciński

Allow to include files from another projects

This adds `project:, file:, ref:` specification support.
parent b97b85c3
...@@ -8,7 +8,7 @@ class Projects::Ci::LintsController < Projects::ApplicationController ...@@ -8,7 +8,7 @@ class Projects::Ci::LintsController < Projects::ApplicationController
def create def create
@content = params[:content] @content = params[:content]
@error = Gitlab::Ci::YamlProcessor.validation_message(@content, yaml_processor_options) @error = Gitlab::Ci::YamlProcessor.validation_message(@content, yaml_processor_options)
@status = @error.blank? @status = @error.blank?
if @error.blank? if @error.blank?
...@@ -24,6 +24,10 @@ class Projects::Ci::LintsController < Projects::ApplicationController ...@@ -24,6 +24,10 @@ class Projects::Ci::LintsController < Projects::ApplicationController
private private
def yaml_processor_options def yaml_processor_options
{ project: @project, sha: project.repository.commit.sha } {
project: @project,
user: current_user,
sha: project.repository.commit.sha
}
end end
end end
...@@ -10,16 +10,16 @@ module BlobViewer ...@@ -10,16 +10,16 @@ module BlobViewer
self.file_types = %i(gitlab_ci) self.file_types = %i(gitlab_ci)
self.binary = false self.binary = false
def validation_message(project, sha) def validation_message(opts)
return @validation_message if defined?(@validation_message) return @validation_message if defined?(@validation_message)
prepare! prepare!
@validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, { project: project, sha: sha }) @validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, opts)
end end
def valid?(project, sha) def valid?(opts)
validation_message(project, sha).blank? validation_message(opts).blank?
end end
end end
end end
...@@ -496,7 +496,7 @@ module Ci ...@@ -496,7 +496,7 @@ module Ci
return @config_processor if defined?(@config_processor) return @config_processor if defined?(@config_processor)
@config_processor ||= begin @config_processor ||= begin
::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha }) ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha, user: user })
rescue Gitlab::Ci::YamlProcessor::ValidationError => e rescue Gitlab::Ci::YamlProcessor::ValidationError => e
self.yaml_errors = e.message self.yaml_errors = e.message
nil nil
......
- if viewer.valid?(@project, @commit.sha) - if viewer.valid?(project: @project, sha: @commit.sha, user: @current_user)
= icon('check fw') = icon('check fw')
This GitLab CI configuration is valid. This GitLab CI configuration is valid.
- else - else
= icon('warning fw') = icon('warning fw')
This GitLab CI configuration is invalid: This GitLab CI configuration is invalid:
= viewer.validation_message(@project, @commit.sha) = viewer.validation_message(project: @project, sha: @commit.sha, user: @current_user)
= link_to 'Learn more', help_page_path('ci/yaml/README') = link_to 'Learn more', help_page_path('ci/yaml/README')
---
title: Allow to include files from another projects in gitlab-ci.yml
merge_request: 24101
author:
type: added
...@@ -1649,7 +1649,7 @@ test: ...@@ -1649,7 +1649,7 @@ test:
> Behaviour expanded in GitLab 10.8 to allow more flexible overriding. > Behaviour expanded in GitLab 10.8 to allow more flexible overriding.
> [Moved](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21603) > [Moved](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21603)
to GitLab Core in 11.4 to GitLab Core in 11.4
> In GitLab 11.7, support for including [GitLab-supplied templates](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/lib/gitlab/ci/templates) directly [was added](https://gitlab.com/gitlab-org/gitlab-ce/issues/53445). > In GitLab 11.7, support for [including GitLab-supplied templates directly](https://gitlab.com/gitlab-org/gitlab-ce/issues/53445) and support for [including templates from another repository](https://gitlab.com/gitlab-org/gitlab-ce/issues/53903) was added.
Using the `include` keyword, you can allow the inclusion of external YAML files. Using the `include` keyword, you can allow the inclusion of external YAML files.
...@@ -1724,7 +1724,7 @@ include: ...@@ -1724,7 +1724,7 @@ include:
--- ---
`include` supports three types of files: `include` supports four types of files:
- **local** to the same repository, referenced by using full paths in the same - **local** to the same repository, referenced by using full paths in the same
repository, with `/` being the root directory. For example: repository, with `/` being the root directory. For example:
...@@ -1750,6 +1750,32 @@ include: ...@@ -1750,6 +1750,32 @@ include:
NOTE: **Note:** NOTE: **Note:**
We don't support the inclusion of local files through Git submodules paths. We don't support the inclusion of local files through Git submodules paths.
- **file** from another repository, referenced by using full paths in the same
repository, with `/` being the root directory. For example:
```yaml
include:
project: 'my-group/my-project'
file: '/templates/.gitlab-ci-template.yml'
```
You can also specify `ref:`. The default `ref:` is the `HEAD` of the project:
```yaml
include:
- project: 'my-group/my-project'
ref: master
file: '/templates/.gitlab-ci-template.yml'
- project: 'my-group/my-project'
ref: v1.0.0
file: '/templates/.gitlab-ci-template.yml'
- project: 'my-group/my-project'
ref: 787123b47f14b552955ca2786bc9542ae66fee5b # git sha
file: '/templates/.gitlab-ci-template.yml'
```
- **remote** in a different location, accessed using HTTP/HTTPS, referenced - **remote** in a different location, accessed using HTTP/HTTPS, referenced
using the full URL. For example: using the full URL. For example:
......
...@@ -8,7 +8,8 @@ module API ...@@ -8,7 +8,8 @@ module API
requires :content, type: String, desc: 'Content of .gitlab-ci.yml' requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
end end
post '/lint' do post '/lint' do
error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
user: current_user)
status 200 status 200
......
...@@ -8,9 +8,9 @@ module Gitlab ...@@ -8,9 +8,9 @@ module Gitlab
class Config class Config
ConfigError = Class.new(StandardError) ConfigError = Class.new(StandardError)
def initialize(config, opts = {}) def initialize(config, project: nil, sha: nil, user: nil)
@config = Config::Extendable @config = Config::Extendable
.new(build_config(config, opts)) .new(build_config(config, project: project, sha: sha, user: user))
.to_hash .to_hash
@global = Entry::Global.new(@config) @global = Entry::Global.new(@config)
...@@ -70,20 +70,21 @@ module Gitlab ...@@ -70,20 +70,21 @@ module Gitlab
private private
def build_config(config, opts = {}) def build_config(config, project:, sha:, user:)
initial_config = Gitlab::Config::Loader::Yaml.new(config).load! initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
project = opts.fetch(:project, nil)
if project if project
process_external_files(initial_config, project, opts) process_external_files(initial_config, project: project, sha: sha, user: user)
else else
initial_config initial_config
end end
end end
def process_external_files(config, project, opts) def process_external_files(config, project:, sha:, user:)
sha = opts.fetch(:sha) { project.repository.root_ref_sha } Config::External::Processor.new(config,
Config::External::Processor.new(config, project: project, sha: sha).perform project: project,
sha: sha || project.repository.root_ref_sha,
user: user).perform
end end
end end
end end
......
...@@ -12,7 +12,7 @@ module Gitlab ...@@ -12,7 +12,7 @@ module Gitlab
YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
Context = Struct.new(:project, :sha) Context = Struct.new(:project, :sha, :user)
def initialize(params, context) def initialize(params, context)
@params = params @params = params
......
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module External
module File
class Project < Base
include Gitlab::Utils::StrongMemoize
attr_reader :project_name, :ref_name
def initialize(params, context = {})
@location = params[:file]
@project_name = params[:project]
@ref_name = params[:ref] || 'HEAD'
super
end
def matching?
super && project_name.present?
end
def content
strong_memoize(:content) { fetch_local_content }
end
private
def validate_content!
if !can_access_local_content?
errors.push("Project `#{project_name}` not found or access denied!")
elsif sha.nil?
errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
elsif content.nil?
errors.push("Project `#{project_name}` file `#{location}` does not exist!")
elsif content.blank?
errors.push("Project `#{project_name}` file `#{location}` is empty!")
end
end
def project
strong_memoize(:project) do
::Project.find_by_full_path(project_name)
end
end
def can_access_local_content?
Ability.allowed?(context.user, :download_code, project)
end
def fetch_local_content
return unless can_access_local_content?
return unless sha
project.repository.blob_data_at(sha, location)
rescue GRPC::NotFound, GRPC::Internal
nil
end
def sha
strong_memoize(:sha) do
project.commit(ref_name).try(:sha)
end
end
end
end
end
end
end
end
...@@ -10,15 +10,17 @@ module Gitlab ...@@ -10,15 +10,17 @@ module Gitlab
FILE_CLASSES = [ FILE_CLASSES = [
External::File::Remote, External::File::Remote,
External::File::Template, External::File::Template,
External::File::Local External::File::Local,
External::File::Project
].freeze ].freeze
AmbigiousSpecificationError = Class.new(StandardError) AmbigiousSpecificationError = Class.new(StandardError)
def initialize(values, project:, sha:) def initialize(values, project:, sha:, user:)
@locations = Array.wrap(values.fetch(:include, [])) @locations = Array.wrap(values.fetch(:include, []))
@project = project @project = project
@sha = sha @sha = sha
@user = user
end end
def process def process
...@@ -61,7 +63,7 @@ module Gitlab ...@@ -61,7 +63,7 @@ module Gitlab
def context def context
strong_memoize(:context) do strong_memoize(:context) do
External::File::Base::Context.new(project, sha) External::File::Base::Context.new(project, sha, user)
end end
end end
end end
......
...@@ -7,9 +7,9 @@ module Gitlab ...@@ -7,9 +7,9 @@ module Gitlab
class Processor class Processor
IncludeError = Class.new(StandardError) IncludeError = Class.new(StandardError)
def initialize(values, project:, sha:) def initialize(values, project:, sha:, user:)
@values = values @values = values
@external_files = External::Mapper.new(values, project: project, sha: sha).process @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
@content = {} @content = {}
rescue External::Mapper::AmbigiousSpecificationError => e rescue External::Mapper::AmbigiousSpecificationError => e
raise IncludeError, e.message raise IncludeError, e.message
......
...@@ -10,7 +10,7 @@ module Gitlab ...@@ -10,7 +10,7 @@ module Gitlab
attr_reader :cache, :stages, :jobs attr_reader :cache, :stages, :jobs
def initialize(config, opts = {}) def initialize(config, opts = {})
@ci_config = Gitlab::Ci::Config.new(config, opts) @ci_config = Gitlab::Ci::Config.new(config, **opts)
@config = @ci_config.to_hash @config = @ci_config.to_hash
unless @ci_config.valid? unless @ci_config.valid?
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'fast_spec_helper' require 'fast_spec_helper'
describe Gitlab::Ci::Config::External::File::Base do describe Gitlab::Ci::Config::External::File::Base do
let(:context) { described_class::Context.new(nil, 'HEAD') } let(:context) { described_class::Context.new(nil, 'HEAD', nil) }
let(:test_class) do let(:test_class) do
Class.new(described_class) do Class.new(described_class) do
......
...@@ -5,7 +5,7 @@ require 'spec_helper' ...@@ -5,7 +5,7 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Local do describe Gitlab::Ci::Config::External::File::Local do
set(:project) { create(:project, :repository) } set(:project) { create(:project, :repository) }
let(:context) { described_class::Context.new(project, '12345') } let(:context) { described_class::Context.new(project, '12345', nil) }
let(:params) { { local: location } } let(:params) { { local: location } }
let(:local_file) { described_class.new(params, context) } let(:local_file) { described_class.new(params, context) }
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Project do
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:context_user) { user }
let(:context) { described_class::Context.new(nil, '12345', context_user) }
let(:subject) { described_class.new(params, context) }
before do
project.add_developer(user)
end
describe '#matching?' do
context 'when a file and project is specified' do
let(:params) { { file: 'file.yml', project: 'project' } }
it 'should return true' do
expect(subject).to be_matching
end
end
context 'with only file is specified' do
let(:params) { { file: 'file.yml' } }
it 'should return false' do
expect(subject).not_to be_matching
end
end
context 'with only project is specified' do
let(:params) { { project: 'project' } }
it 'should return false' do
expect(subject).not_to be_matching
end
end
context 'with a missing local key' do
let(:params) { {} }
it 'should return false' do
expect(subject).not_to be_matching
end
end
end
describe '#valid?' do
context 'when a valid path is used' do
let(:params) do
{ project: project.full_path, file: '/file.yml' }
end
let(:root_ref_sha) { project.repository.root_ref_sha }
before do
stub_project_blob(root_ref_sha, '/file.yml') { 'image: ruby:2.1' }
end
it 'should return true' do
expect(subject).to be_valid
end
context 'when user does not have permission to access file' do
let(:context_user) { create(:user) }
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` not found or access denied!")
end
end
end
context 'when a valid path with custom ref is used' do
let(:params) do
{ project: project.full_path, ref: 'master', file: '/file.yml' }
end
let(:ref_sha) { project.commit('master').sha }
before do
stub_project_blob(ref_sha, '/file.yml') { 'image: ruby:2.1' }
end
it 'should return true' do
expect(subject).to be_valid
end
end
context 'when an empty file is used' do
let(:params) do
{ project: project.full_path, file: '/file.yml' }
end
let(:root_ref_sha) { project.repository.root_ref_sha }
before do
stub_project_blob(root_ref_sha, '/file.yml') { '' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/file.yml` is empty!")
end
end
context 'when non-existing ref is used' do
let(:params) do
{ project: project.full_path, ref: 'I-Do-Not-Exist', file: '/file.yml' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` reference `I-Do-Not-Exist` does not exist!")
end
end
context 'when non-existing file is requested' do
let(:params) do
{ project: project.full_path, file: '/invalid-file.yml' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/invalid-file.yml` does not exist!")
end
end
context 'when file is not a yaml file' do
let(:params) do
{ project: project.full_path, file: '/invalid-file' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
end
end
end
private
def stub_project_blob(ref, path)
allow_any_instance_of(Repository)
.to receive(:blob_data_at)
.with(ref, path) { yield }
end
end
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Remote do describe Gitlab::Ci::Config::External::File::Remote do
let(:context) { described_class::Context.new(nil, '12345') } let(:context) { described_class::Context.new(nil, '12345', nil) }
let(:params) { { remote: location } } let(:params) { { remote: location } }
let(:remote_file) { described_class.new(params, context) } let(:remote_file) { described_class.new(params, context) }
let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
......
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::Mapper do describe Gitlab::Ci::Config::External::Mapper do
set(:project) { create(:project, :repository) } set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' } let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' }
let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
...@@ -20,7 +21,7 @@ describe Gitlab::Ci::Config::External::Mapper do ...@@ -20,7 +21,7 @@ describe Gitlab::Ci::Config::External::Mapper do
end end
describe '#process' do describe '#process' do
subject { described_class.new(values, project: project, sha: '123456').process } subject { described_class.new(values, project: project, sha: '123456', user: user).process }
context "when single 'include' keyword is defined" do context "when single 'include' keyword is defined" do
context 'when the string is a local file' do context 'when the string is a local file' do
......
...@@ -4,8 +4,13 @@ require 'spec_helper' ...@@ -4,8 +4,13 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::Processor do describe Gitlab::Ci::Config::External::Processor do
set(:project) { create(:project, :repository) } set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:processor) { described_class.new(values, project: project, sha: '12345') } let(:processor) { described_class.new(values, project: project, sha: '12345', user: user) }
before do
project.add_developer(user)
end
describe "#perform" do describe "#perform" do
context 'when no external files defined' do context 'when no external files defined' do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Ci::Config do describe Gitlab::Ci::Config do
set(:user) { create(:user) }
let(:config) do let(:config) do
described_class.new(yml) described_class.new(yml, project: nil, sha: nil, user: nil)
end end
context 'when config is valid' do context 'when config is valid' do
...@@ -154,7 +156,7 @@ describe Gitlab::Ci::Config do ...@@ -154,7 +156,7 @@ describe Gitlab::Ci::Config do
end end
let(:config) do let(:config) do
described_class.new(gitlab_ci_yml, project: project, sha: '12345') described_class.new(gitlab_ci_yml, project: project, sha: '12345', user: user)
end end
before do before do
...@@ -228,7 +230,7 @@ describe Gitlab::Ci::Config do ...@@ -228,7 +230,7 @@ describe Gitlab::Ci::Config do
expect(project.repository).to receive(:blob_data_at) expect(project.repository).to receive(:blob_data_at)
.with('eeff1122', local_location) .with('eeff1122', local_location)
described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122') described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122', user: user)
end end
end end
...@@ -236,7 +238,7 @@ describe Gitlab::Ci::Config do ...@@ -236,7 +238,7 @@ describe Gitlab::Ci::Config do
it 'is using latest SHA on the default branch' do it 'is using latest SHA on the default branch' do
expect(project.repository).to receive(:root_ref_sha) expect(project.repository).to receive(:root_ref_sha)
described_class.new(gitlab_ci_yml, project: project) described_class.new(gitlab_ci_yml, project: project, sha: nil, user: user)
end end
end end
end end
......
...@@ -3,10 +3,10 @@ require 'spec_helper' ...@@ -3,10 +3,10 @@ require 'spec_helper'
module Gitlab module Gitlab
module Ci module Ci
describe YamlProcessor do describe YamlProcessor do
subject { described_class.new(config) } subject { described_class.new(config, user: nil) }
describe '#build_attributes' do describe '#build_attributes' do
subject { described_class.new(config).build_attributes(:rspec) } subject { described_class.new(config, user: nil).build_attributes(:rspec) }
describe 'attributes list' do describe 'attributes list' do
let(:config) do let(:config) do
......
...@@ -4,7 +4,9 @@ describe BlobViewer::GitlabCiYml do ...@@ -4,7 +4,9 @@ describe BlobViewer::GitlabCiYml do
include FakeBlobHelpers include FakeBlobHelpers
include RepoHelpers include RepoHelpers
let(:project) { create(:project, :repository) } set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:data) { File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) } let(:data) { File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) }
let(:blob) { fake_blob(path: '.gitlab-ci.yml', data: data) } let(:blob) { fake_blob(path: '.gitlab-ci.yml', data: data) }
let(:sha) { sample_commit.id } let(:sha) { sample_commit.id }
...@@ -14,12 +16,12 @@ describe BlobViewer::GitlabCiYml do ...@@ -14,12 +16,12 @@ describe BlobViewer::GitlabCiYml do
it 'calls prepare! on the viewer' do it 'calls prepare! on the viewer' do
expect(subject).to receive(:prepare!) expect(subject).to receive(:prepare!)
subject.validation_message(project, sha) subject.validation_message(project: project, sha: sha, user: user)
end end
context 'when the configuration is valid' do context 'when the configuration is valid' do
it 'returns nil' do it 'returns nil' do
expect(subject.validation_message(project, sha)).to be_nil expect(subject.validation_message(project: project, sha: sha, user: user)).to be_nil
end end
end end
...@@ -27,7 +29,7 @@ describe BlobViewer::GitlabCiYml do ...@@ -27,7 +29,7 @@ describe BlobViewer::GitlabCiYml do
let(:data) { 'oof' } let(:data) { 'oof' }
it 'returns the error message' do it 'returns the error message' do
expect(subject.validation_message(project, sha)).to eq('Invalid configuration format') expect(subject.validation_message(project: project, sha: sha, user: user)).to eq('Invalid configuration format')
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment