Merge branch '20868-include-external-files-in-gitlab-ci-yml' into 'master'

20868 - Includes External Files in Gitlab CI YML

See merge request gitlab-org/gitlab-ee!4262

app/models/blob_viewer/gitlab_ci_yml.rb

@@ -8,16 +8,16 @@ module BlobViewer
     self.file_types = %i(gitlab_ci)
     self.binary = false
 
-    def validation_message
+    def validation_message(project, sha)
       return @validation_message if defined?(@validation_message)
 
       prepare!
 
-      @validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data)
+      @validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, { project: project, sha: sha })
     end
 
-    def valid?
-      validation_message.blank?
+    def valid?(project, sha)
+      validation_message(project, sha).blank?
     end
   end
 end

app/models/ci/pipeline.rb

@@ -406,7 +406,7 @@ module Ci
       return @config_processor if defined?(@config_processor)
 
       @config_processor ||= begin
-        Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
+        initialize_yaml_processor
       rescue Gitlab::Ci::YamlProcessor::ValidationError, Psych::SyntaxError => e
         self.yaml_errors = e.message
         nil
@@ -416,6 +416,10 @@ module Ci
       end
     end
 
+    def initialize_yaml_processor
+      Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
+    end
+
     def ci_yaml_file_path
       if project.ci_config_path.blank?
         '.gitlab-ci.yml'

app/models/repository.rb

@@ -1061,6 +1061,14 @@ class Repository
                                        remote_branch: merge_request.target_branch)
   end
 
+  def blob_data_at(sha, path)
+    blob = blob_at(sha, path)
+    return unless blob
+
+    blob.load_all_data!
+    blob.data
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid
@@ -1075,14 +1083,6 @@ class Repository
     ::Commit.new(commit, @project) if commit
   end
 
-  def blob_data_at(sha, path)
-    blob = blob_at(sha, path)
-    return unless blob
-
-    blob.load_all_data!
-    blob.data
-  end
-
   def cache
     # TODO: should we use UUIDs here? We could move repositories without clearing this cache
     @cache ||= RepositoryCache.new(full_path, @project.id)

app/views/projects/blob/viewers/_gitlab_ci_yml.html.haml

-- if viewer.valid?
+- if viewer.valid?(@project, @commit.sha)
   = icon('check fw')
   This GitLab CI configuration is valid.
 - else
   = icon('warning fw')
   This GitLab CI configuration is invalid:
-  = viewer.validation_message
+  = viewer.validation_message(@project, @commit.sha)
 
 = link_to 'Learn more', help_page_path('ci/yaml/README')

changelogs/unreleased-ee/20868-include-external-files-in-gitlab-ci-yml.yml

+---
+title: Dry up CI/CD gitlab-ci.yml configuration by allowing inclusion of external files
+merge_request: 4262
+author:
+type: added

doc/ci/yaml/README.md

@@ -361,6 +361,152 @@ Additionally, if you have a job that unconditionally recreates the cache without
 reference to its previous contents, you can use `policy: push` in that job to
 skip the download step.
 
+### include (EEP)
+
+> Introduced in [GitLab Enterprise Edition Premium][ee] 10.5.
+
+From 10.5 we can use `include` keyword to allow the inclusion of external YAML files.
+
+```yaml
+# Content of https://gitlab.com/awesome-project/raw/master/.before-script-template.yml
+before_script:
+  - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+```
+
+```yaml
+# Content of .gitlab-ci.yml
+include: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml'
+
+rspec:
+  script:
+    - bundle exec rspec
+
+rubocop:
+  script:
+    - bundle exec rubocop
+```
+
+In the above example `.before-script-template.yml` content will be automatically fetched and evaluated along with the content of `.gitlab-ci.yml`.
+
+`include` supports two types of files:
+
+- **local** to the same repository, referenced using the paths in the same the repository, i.e:
+
+```yaml
+# Within the repository
+include: '/templates/.gitlab-ci-template.yml'
+```
+
+- **remote** in a different location, accessed using HTTP/HTTPS protocol, referenced using the full URL, i.e:
+
+```yaml
+include: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml'
+```
+
+Also, `include` supports a single string or an array composed by different values, so
+
+```yaml
+include: '/templates/.gitlab-ci-template.yml'
+```
+
+and
+
+```yaml
+include:
+  - 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml'
+  - '/templates/.gitlab-ci-template.yml'
+```
+
+are both valid use cases.
+
+#### Restrictions
+
+- We can only use files that are currently tracked by Git on the same branch your configuration file is. In other words, when using a **local file** make sure that both, `.gitlab-ci.yml` and the local file are on the same branch.
+- Since external files defined on `include` are evaluated first, the content on `.gitlab-ci.yml` **will always take precedence over the content of the external files, no matters of the position of the `include` keyword, allowing to override values and functions with local definitions**, for example:
+
+```yaml
+# Content of https://company.com/autodevops-template.yml
+
+variables:
+  POSTGRES_USER: user
+  POSTGRES_PASSWORD: testing_password
+  POSTGRES_ENABLED: "true"
+  POSTGRES_DB: $CI_ENVIRONMENT_SLUG
+
+  KUBERNETES_VERSION: 1.8.6
+  HELM_VERSION: 2.6.1
+  CODECLIMATE_VERSION: 0.69.0
+
+production:
+  stage: production
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - install_tiller
+    - create_secret
+    - deploy
+    - delete canary
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN
+  only:
+    refs:
+      - master
+    kubernetes: active
+```
+
+```yaml
+# Content of .gitlab-ci.yml
+
+include: 'https://company.com/autodevops-template.yml'
+
+image: alpine:latest
+
+variables:
+  POSTGRES_USER: root
+  POSTGRES_PASSWORD: secure_password
+  POSTGRES_DB: company_database
+
+stages:
+  - build
+  - test
+  - review
+  - dast
+  - staging
+  - canary
+  - production
+  - performance
+  - cleanup
+
+production:
+  stage: production
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - install_tiller
+    - create_secret
+    - deploy
+  environment:
+    name: production
+    url: http://auto_devops_domain.com
+  only:
+    refs:
+      - master
+
+# ....
+```
+
+In this case, the variables `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` along with the `production` job defined on `autodevops-template.yml` will be overridden by the ones defined on `.gitlab-ci.yml`.
+
 ## Jobs
 
 `.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job
@@ -1615,3 +1761,4 @@ CI with various languages.
 [ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
 [ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442
 [schedules]: ../../user/project/pipelines/schedules.md
+[ee]: https://about.gitlab.com/gitlab-ee/

ee/app/models/ee/ci/pipeline.rb

@@ -32,6 +32,10 @@ module EE
       def dast_artifact
         artifacts.dast.find(&:has_dast_json?)
       end
+
+      def initialize_yaml_processor
+        ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha })
+      end
     end
   end
 end

ee/app/models/license.rb

@@ -39,6 +39,7 @@ class License < ActiveRecord::Base
     db_load_balancing
     deploy_board
     extended_audit_events
+    external_files_in_gitlab_ci
     file_locks
     geo
     group_issue_boards

ee/lib/ee/gitlab/ci/config.rb

+module EE
+  module Gitlab
+    module Ci
+      #
+      # EE Base GitLab CI configuration facade
+      #
+      module Config
+        def initialize(config, opts = {})
+          super
+        rescue ::Gitlab::Ci::External::Processor::FileError => e
+          raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message
+        end
+
+        private
+
+        def build_config(config, opts = {})
+          initial_config = ::Gitlab::Ci::Config::Loader.new(config).load!
+          project = opts.fetch(:project, nil)
+
+          if project&.feature_available?(:external_files_in_gitlab_ci)
+            process_external_files(initial_config, project, opts)
+          elsif initial_config.include?(:include)
+            raise ::Gitlab::Ci::YamlProcessor::ValidationError, "Your license does not allow to use 'include' keyword in CI/CD configuration file"
+          else
+            initial_config
+          end
+        end
+
+        def process_external_files(config, project, opts)
+          sha = opts.fetch(:sha, project.repository.commit.sha)
+          ::Gitlab::Ci::External::Processor.new(config, project, sha).perform
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/external/file/local.rb

+module Gitlab
+  module Ci
+    module External
+      module File
+        class Local
+          attr_reader :location, :project, :sha
+
+          def initialize(location, opts = {})
+            @location = location
+            @project = opts.fetch(:project)
+            @sha = opts.fetch(:sha)
+          end
+
+          def valid?
+            local_file_content
+          end
+
+          def content
+            local_file_content
+          end
+
+          private
+
+          def local_file_content
+            @local_file_content ||= project.repository.blob_data_at(sha, location)
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/external/file/remote.rb

+module Gitlab
+  module Ci
+    module External
+      module File
+        class Remote
+          include Gitlab::Utils::StrongMemoize
+          attr_reader :location
+
+          def initialize(location, opts = {})
+            @location = location
+          end
+
+          def valid?
+            ::Gitlab::UrlSanitizer.valid?(location) && content
+          end
+
+          def content
+            return @content if defined?(@content)
+
+            @content = strong_memoize(:content) do
+              begin
+                HTTParty.get(location)
+              rescue HTTParty::Error, Timeout::Error
+                false
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/external/mapper.rb

+module Gitlab
+  module Ci
+    module External
+      class Mapper
+        def initialize(values, project, sha)
+          @locations = Array(values.fetch(:include, []))
+          @project = project
+          @sha = sha
+        end
+
+        def process
+          locations.map { |location| build_external_file(location) }
+        end
+
+        private
+
+        attr_reader :locations, :project, :sha
+
+        def build_external_file(location)
+          remote_file = Gitlab::Ci::External::File::Remote.new(location)
+
+          if remote_file.valid?
+            remote_file
+          else
+            options = { project: project, sha: sha }
+            Gitlab::Ci::External::File::Local.new(location, options)
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/external/processor.rb

+module Gitlab
+  module Ci
+    module External
+      class Processor
+        FileError = Class.new(StandardError)
+
+        def initialize(values, project, sha)
+          @values = values
+          @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process
+          @content = {}
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            @content.merge!(content_of(external_file))
+          end
+
+          append_inline_content
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files, :content
+
+        def validate_external_file(external_file)
+          unless external_file.valid?
+            raise FileError, "External file: '#{external_file.location}' should be a valid local or remote file"
+          end
+        end
+
+        def content_of(external_file)
+          Gitlab::Ci::Config::Loader.new(external_file.content).load!
+        end
+
+        def append_inline_content
+          @content.merge!(@values)
+        end
+
+        def remove_include_keyword
+          content.delete(:include)
+          content
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config.rb

 module Gitlab
   module Ci
-    ##
+    #
     # Base GitLab CI Configuration facade
     #
     class Config
-      def initialize(config)
-        @config = Loader.new(config).load!
+      prepend EE::Gitlab::Ci::Config
 
+      def initialize(config, opts = {})
+        @config = build_config(config, opts)
         @global = Entry::Global.new(@config)
         @global.compose!
       end
@@ -57,6 +58,13 @@ module Gitlab
       def jobs
         @global.jobs_value
       end
+
+      private
+
+      # 'Opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
+      def build_config(config, opts = {})
+        Loader.new(config).load!
+      end
     end
   end
 end

lib/gitlab/ci/yaml_processor.rb

@@ -7,8 +7,8 @@ module Gitlab
 
       attr_reader :cache, :stages, :jobs
 
-      def initialize(config)
-        @ci_config = Gitlab::Ci::Config.new(config)
+      def initialize(config, opts = {})
+        @ci_config = Gitlab::Ci::Config.new(config, opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
@@ -79,11 +79,11 @@ module Gitlab
         seeds.compact
       end
 
-      def self.validation_message(content)
+      def self.validation_message(content, opts = {})
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
         begin
-          Gitlab::Ci::YamlProcessor.new(content)
+          Gitlab::Ci::YamlProcessor.new(content, opts)
           nil
         rescue ValidationError, Psych::SyntaxError => e
           e.message

spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml

+before_script:
+  - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+rspec:
+  script:
+    - bundle exec rspec

spec/ee/spec/lib/ee/gitlab/ci/config_spec.rb

+require 'spec_helper'
+
+describe EE::Gitlab::Ci::Config do
+  let(:project) { create(:project, :repository) }
+  let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+  let(:gitlab_ci_yml) do
+    <<~HEREDOC
+    include:
+      - /spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml
+      - #{remote_location}
+
+    image: ruby:2.2
+    HEREDOC
+  end
+  let(:config) { ::Gitlab::Ci::Config.new(gitlab_ci_yml, { project: project, sha: '12345' }) }
+
+  context 'when the project does not have EEP license' do
+    before do
+      allow(project).to receive(:feature_available?).with(:external_files_in_gitlab_ci).and_return(false)
+    end
+
+    it "should raise a ValidationError" do
+      expect { config }.to raise_error(
+        ::Gitlab::Ci::YamlProcessor::ValidationError,
+        "Your license does not allow to use 'include' keyword in CI/CD configuration file"
+      )
+    end
+  end
+
+  context 'when the project has EEP license' do
+    let(:remote_file_content) do
+      <<~HEREDOC
+      variables:
+        AUTO_DEVOPS_DOMAIN: domain.example.com
+        POSTGRES_USER: user
+        POSTGRES_PASSWORD: testing-password
+        POSTGRES_ENABLED: "true"
+        POSTGRES_DB: $CI_ENVIRONMENT_SLUG
+      HEREDOC
+    end
+    let(:local_file_content) {  File.read("#{Rails.root}/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml") }
+
+    before do
+      allow(project).to receive(:feature_available?).with(:external_files_in_gitlab_ci).and_return(true)
+    end
+
+    context "when gitlab_ci_yml has valid 'include' defined" do
+      before do
+        allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:local_file_content).and_return(local_file_content)
+        WebMock.stub_request(:get, remote_location).to_return(body: remote_file_content)
+      end
+
+      it 'should return a composed hash' do
+        before_script_values = [
+          "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v",
+          "which ruby",
+          "gem install bundler --no-ri --no-rdoc",
+          "bundle install --jobs $(nproc)  \"${FLAGS[@]}\""
+        ]
+        variables = {
+          AUTO_DEVOPS_DOMAIN: "domain.example.com",
+          POSTGRES_USER: "user",
+          POSTGRES_PASSWORD: "testing-password",
+          POSTGRES_ENABLED: "true",
+          POSTGRES_DB: "$CI_ENVIRONMENT_SLUG"
+        }
+        composed_hash = {
+          before_script: before_script_values,
+          image: "ruby:2.2",
+          rspec: { script: ["bundle exec rspec"] },
+          variables: variables
+        }
+
+        expect(config.to_hash).to eq(composed_hash)
+      end
+    end
+
+    context "when gitlab_ci.yml has invalid 'include' defined"  do
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include: invalid
+        HEREDOC
+      end
+
+      it 'raises error YamlProcessor validationError' do
+        expect { config }.to raise_error(
+          ::Gitlab::Ci::YamlProcessor::ValidationError,
+          "External file: 'invalid' should be a valid local or remote file"
+        )
+      end
+    end
+
+    context "when both external files and gitlab_ci.yml defined the same key" do
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include:
+          - #{remote_location}
+
+        image: ruby:2.2
+        HEREDOC
+      end
+
+      let(:remote_file_content) do
+        <<~HEREDOC
+        image: php:5-fpm-alpine
+        HEREDOC
+      end
+
+      it 'should take precedence' do
+        WebMock.stub_request(:get, remote_location).to_return(body: remote_file_content)
+        expect(config.to_hash).to eq({ image: 'ruby:2.2' })
+      end
+    end
+  end
+end

spec/ee/spec/lib/gitlab/ci/external/file/local_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::External::File::Local do
+  let(:project) { create(:project, :repository) }
+  let(:local_file) { described_class.new(location, { project: project, sha: '12345' }) }
+
+  describe "#valid?" do
+    context 'when is a valid local path' do
+      let(:location) { '/vendor/gitlab-ci-yml/existent-file.yml' }
+
+      before do
+        allow_any_instance_of(described_class).to receive(:local_file_content).and_return("image: 'ruby2:2'")
+      end
+
+      it 'should return true' do
+        expect(local_file.valid?).to be_truthy
+      end
+    end
+
+    context 'when is not a valid local path' do
+      let(:location) { '/vendor/gitlab-ci-yml/non-existent-file.yml' }
+
+      it 'should return false' do
+        expect(local_file.valid?).to be_falsy
+      end
+    end
+
+    describe "#content" do
+      let(:local_file_content) do
+        <<~HEREDOC
+      before_script:
+        - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+        - ruby -v
+        - which ruby
+        - gem install bundler --no-ri --no-rdoc
+        - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+        HEREDOC
+      end
+
+      context 'with a local file' do
+        let(:location) { '/vendor/gitlab-ci-yml/non-existent-file.yml' }
+
+        before do
+          allow_any_instance_of(described_class).to receive(:local_file_content).and_return(local_file_content)
+        end
+
+        it 'should return the content of the file' do
+          expect(local_file.content).to eq(local_file_content)
+        end
+      end
+    end
+  end
+end

spec/ee/spec/lib/gitlab/ci/external/file/remote_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::External::File::Remote do
+  let(:remote_file) { described_class.new(location) }
+  let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+  let(:remote_file_content) do
+    <<~HEREDOC
+      before_script:
+        - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+        - ruby -v
+        - which ruby
+        - gem install bundler --no-ri --no-rdoc
+        - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+    HEREDOC
+  end
+
+  describe "#valid?" do
+    context 'when is a valid remote url' do
+      before do
+        WebMock.stub_request(:get, location).to_return(body: remote_file_content)
+      end
+
+      it 'should return true' do
+        expect(remote_file.valid?).to be_truthy
+      end
+    end
+
+    context 'when is not a valid remote url' do
+      let(:location) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      it 'should return false' do
+        expect(remote_file.valid?).to be_falsy
+      end
+    end
+
+    context 'with a timeout' do
+      before do
+        allow(HTTParty).to receive(:get).and_raise(Timeout::Error)
+      end
+
+      it 'should be falsy' do
+        expect(remote_file.valid?).to be_falsy
+      end
+    end
+  end
+
+  describe "#content" do
+    context 'with a valid remote file' do
+      before do
+        WebMock.stub_request(:get, location).to_return(body: remote_file_content)
+      end
+
+      it 'should return the content of the file' do
+        expect(remote_file.content).to eql(remote_file_content)
+      end
+    end
+
+    context 'with a timeout' do
+      before do
+        allow(HTTParty).to receive(:get).and_raise(Timeout::Error)
+      end
+
+      it 'should be falsy' do
+        expect(remote_file.content).to be_falsy
+      end
+    end
+  end
+end

spec/ee/spec/lib/gitlab/ci/external/mapper_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::External::Mapper do
+  let(:project) { create(:project, :repository) }
+  let(:file_content) do
+    <<~HEREDOC
+    image: 'ruby:2.2'
+    HEREDOC
+  end
+
+  describe '#process' do
+    subject { described_class.new(values, project, '123456').process }
+
+    context "when 'include' keyword is defined as string" do
+      context 'when the string is a local file' do
+        let(:values) do
+          {
+            include: '/vendor/gitlab-ci-yml/non-existent-file.yml',
+            image: 'ruby:2.2'
+          }
+        end
+
+        it 'returns an array' do
+          expect(subject).to be_an(Array)
+        end
+
+        it 'returns File instances' do
+          expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Local)
+        end
+      end
+
+      context 'when the string is a remote file' do
+        let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+        let(:values) do
+          {
+            include: remote_url,
+            image: 'ruby:2.2'
+          }
+        end
+
+        before do
+          WebMock.stub_request(:get, remote_url).to_return(body: file_content)
+        end
+
+        it 'returns an array' do
+          expect(subject).to be_an(Array)
+        end
+
+        it 'returns File instances' do
+          expect(subject.first).to be_an_instance_of(Gitlab::Ci::External::File::Remote)
+        end
+      end
+    end
+
+    context "when 'include' is defined as an array" do
+      let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+      let(:values) do
+        {
+          include:
+          [
+            remote_url,
+            '/vendor/gitlab-ci-yml/template.yml'
+          ],
+          image: 'ruby:2.2'
+        }
+      end
+
+      before do
+        WebMock.stub_request(:get, remote_url).to_return(body: file_content)
+      end
+
+      it 'returns an array' do
+        expect(subject).to be_an(Array)
+      end
+
+      it 'returns Files instances' do
+        expect(subject).to all(respond_to(:valid?))
+        expect(subject).to all(respond_to(:content))
+      end
+    end
+
+    context "when 'include' is not defined" do
+      let(:values) do
+        {
+          image: 'ruby:2.2'
+        }
+      end
+
+      it 'returns an empty array' do
+        expect(subject).to be_empty
+      end
+    end
+  end
+end

spec/ee/spec/lib/gitlab/ci/external/processor_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::External::Processor do
+  let(:project) { create(:project, :repository) }
+  let(:processor) { described_class.new(values, project, '12345') }
+
+  describe "#perform" do
+    context 'when no external files defined' do
+      let(:values) { { image: 'ruby:2.2' } }
+
+      it 'should return the same values' do
+        expect(processor.perform).to eq(values)
+      end
+    end
+
+    context 'when an invalid local file is defined' do
+      let(:values) { { include: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2' } }
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(
+          described_class::FileError,
+          "External file: '/vendor/gitlab-ci-yml/non-existent-file.yml' should be a valid local or remote file"
+        )
+      end
+    end
+
+    context 'when an invalid remote file is defined' do
+      let(:values) { { include: 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', image: 'ruby:2.2' } }
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(
+          described_class::FileError,
+          "External file: 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' should be a valid local or remote file"
+        )
+      end
+    end
+
+    context 'with a valid remote external file is defined' do
+      let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+      let(:values) { { include: remote_file, image: 'ruby:2.2' } }
+      let(:external_file_content) do
+        <<-HEREDOC
+        before_script:
+          - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+          - ruby -v
+          - which ruby
+          - gem install bundler --no-ri --no-rdoc
+          - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+        rspec:
+          script:
+            - bundle exec rspec
+
+        rubocop:
+          script:
+            - bundle exec rubocop
+        HEREDOC
+      end
+
+      before do
+        WebMock.stub_request(:get, remote_file).to_return(body: external_file_content)
+      end
+
+      it 'should append the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :before_script, :rspec, :rubocop])
+      end
+
+      it "should remove the 'include' keyword" do
+        expect(processor.perform[:include]).to be_nil
+      end
+    end
+
+    context 'with a valid local external file is defined' do
+      let(:values) { { include: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2' } }
+      let(:local_file_content) do
+        <<-HEREDOC
+        before_script:
+          - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+          - ruby -v
+          - which ruby
+          - gem install bundler --no-ri --no-rdoc
+          - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+        HEREDOC
+      end
+
+      before do
+        allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:local_file_content).and_return(local_file_content)
+      end
+
+      it 'should append the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :before_script])
+      end
+
+      it "should remove the 'include' keyword" do
+        expect(processor.perform[:include]).to be_nil
+      end
+    end
+
+    context 'with multiple external files are defined' do
+      let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+      let(:external_files) do
+        [
+          '/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml',
+          remote_file
+        ]
+      end
+      let(:values) do
+        {
+          include: external_files,
+          image: 'ruby:2.2'
+        }
+      end
+
+      let(:remote_file_content) do
+        <<-HEREDOC
+        stages:
+          - build
+          - review
+          - cleanup
+        HEREDOC
+      end
+
+      before do
+        local_file_content = File.read("#{Rails.root}/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml")
+        allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:local_file_content).and_return(local_file_content)
+        WebMock.stub_request(:get, remote_file).to_return(body: remote_file_content)
+      end
+
+      it 'should append the files to the values' do
+        expect(processor.perform.keys).to match_array([:image, :stages, :before_script, :rspec])
+      end
+
+      it "should remove the 'include' keyword" do
+        expect(processor.perform[:include]).to be_nil
+      end
+    end
+
+    context 'when external files are defined but not valid' do
+      let(:values) { { include: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2' } }
+
+      let(:local_file_content) { 'invalid content file ////' }
+
+      before do
+        allow_any_instance_of(Gitlab::Ci::External::File::Local).to receive(:local_file_content).and_return(local_file_content)
+      end
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(Gitlab::Ci::Config::Loader::FormatError)
+      end
+    end
+
+    context "when both external files and values defined the same key" do
+      let(:remote_file) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+      let(:values) do
+        {
+          include: remote_file,
+          image: 'ruby:2.2'
+        }
+      end
+
+      let(:remote_file_content) do
+        <<~HEREDOC
+        image: php:5-fpm-alpine
+        HEREDOC
+      end
+
+      it 'should take precedence' do
+        WebMock.stub_request(:get, remote_file).to_return(body: remote_file_content)
+        expect(processor.perform[:image]).to eq('ruby:2.2')
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config_spec.rb

@@ -49,7 +49,7 @@ describe Gitlab::Ci::Config do
       describe '.new' do
         it 'raises error' do
           expect { config }.to raise_error(
-            Gitlab::Ci::Config::Loader::FormatError,
+            ::Gitlab::Ci::Config::Loader::FormatError,
             /Invalid configuration format/
           )
         end

spec/models/blob_viewer/gitlab_ci_yml_spec.rb

@@ -2,22 +2,24 @@ require 'spec_helper'
 
 describe BlobViewer::GitlabCiYml do
   include FakeBlobHelpers
+  include RepoHelpers
 
-  let(:project) { build_stubbed(:project) }
+  let(:project) { build_stubbed(:project, :repository) }
   let(:data) { File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) }
   let(:blob) { fake_blob(path: '.gitlab-ci.yml', data: data) }
+  let(:sha) { sample_commit.id }
   subject { described_class.new(blob) }
 
   describe '#validation_message' do
     it 'calls prepare! on the viewer' do
       expect(subject).to receive(:prepare!)
 
-      subject.validation_message
+      subject.validation_message(project, sha)
     end
 
     context 'when the configuration is valid' do
       it 'returns nil' do
-        expect(subject.validation_message).to be_nil
+        expect(subject.validation_message(project, sha)).to be_nil
       end
     end
 
@@ -25,7 +27,7 @@ describe BlobViewer::GitlabCiYml do
       let(:data) { 'oof' }
 
       it 'returns the error message' do
-        expect(subject.validation_message).to eq('Invalid configuration format')
+        expect(subject.validation_message(project, sha)).to eq('Invalid configuration format')
       end
     end
   end

spec/models/ci/pipeline_spec.rb

@@ -1434,7 +1434,7 @@ describe Ci::Pipeline, :mailer do
         create(:ci_pipeline, config: { rspec: { script: 'rake test' } })
       end
 
-      it 'does not containyaml errors' do
+      it 'does not contain yaml errors' do
         expect(pipeline).not_to have_yaml_errors
       end
     end