Merge branch 'mc/feature/ci-config-render-api-endpoint' into 'master'

Add endpoint for rendering expanded CI YAML config

See merge request gitlab-org/gitlab!42380

changelogs/unreleased/mc-feature-ci-config-render-api-endpoint.yml

+---
+title: Show expanded CI config in CI lint API endpoint.
+merge_request: 42380
+author:
+type: added

doc/api/lint.md

@@ -4,11 +4,14 @@ group: Continuous Integration
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
 ---
 
-# Validate the `.gitlab-ci.yml` (API)
+# CI Lint API
+
+## Validate the CI YAML config
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953) in GitLab 8.12.
 
-Checks if your `.gitlab-ci.yml` file is valid.
+Checks if CI/CD YAML configuration is valid. This endpoint validates basic CI/CD
+configuration syntax. It doesn't have any namespace specific context.
 
 ```plaintext
 POST /ci/lint
@@ -16,13 +19,15 @@ POST /ci/lint
 
 | Attribute  | Type    | Required | Description |
 | ---------- | ------- | -------- | -------- |
-| `content`  | string    | yes      | the `.gitlab-ci.yaml` content|
+| `content`              | string     | yes      | The CI/CD configuration content. |
+| `include_merged_yaml`  | boolean    | no       | If the [expanded CI/CD configuration](#yaml-expansion) should be included in the response. |
 
 ```shell
 curl --header "Content-Type: application/json" "https://gitlab.example.com/api/v4/ci/lint" --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}'
 ```
 
-Be sure to copy paste the exact contents of `.gitlab-ci.yml` as YAML is very picky about indentation and spaces.
+Be sure to paste the exact contents of your GitLab CI/CD YAML config because YAML
+is very sensitive about indentation and spacing.
 
 Example responses:
 
@@ -53,3 +58,39 @@ Example responses:
     "error": "content is missing"
   }
   ```
+
+### YAML expansion
+
+The expansion only works for CI configurations that don't have local [includes](../ci/yaml/README.md#include).
+
+Example contents of a `.gitlab-ci.yml` passed to the CI Lint API with
+`include_merged_yaml` set as true:
+
+```yaml
+include:
+  remote: 'https://example.com/remote.yaml'
+
+test:
+  stage: test
+  script:
+    - echo 1
+```
+
+Example contents of `https://example.com/remote.yaml`:
+
+```yaml
+another_test:
+  stage: test
+  script:
+    - echo 2
+```
+
+Example response:
+
+```json
+{
+  "status": "valid",
+  "errors": [],
+  "merged_config": "---\n:another_test:\n  :stage: test\n  :script: echo 2\n:test:\n  :stage: test\n  :script: echo 1\n"
+}
+```

lib/api/lint.rb

@@ -6,18 +6,23 @@ module API
       desc 'Validation of .gitlab-ci.yml content'
       params do
         requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
+        optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response'
       end
       post '/lint' do
-        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
-          user: current_user)
+        result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute
+        error = result.errors.first
 
         status 200
 
-        if error.blank?
+        response = if error.blank?
                      { status: 'valid', errors: [] }
                    else
                      { status: 'invalid', errors: [error] }
                    end
+
+        response.tap do |response|
+          response[:merged_yaml] = result.merged_yaml if params[:include_merged_yaml]
+        end
       end
     end
   end

lib/gitlab/ci/yaml_processor/result.rb

@@ -95,6 +95,10 @@ module Gitlab
             }.compact }.compact
         end
 
+        def merged_yaml
+          @ci_config&.to_hash&.to_yaml
+        end
+
         private
 
         def variables

spec/lib/gitlab/ci/yaml_processor/result_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+module Gitlab
+  module Ci
+    class YamlProcessor
+      RSpec.describe Result do
+        include StubRequests
+
+        let(:user) { create(:user) }
+        let(:ci_config) { Gitlab::Ci::Config.new(config_content, user: user) }
+        let(:result) { described_class.new(ci_config: ci_config, warnings: ci_config&.warnings) }
+
+        describe '#merged_yaml' do
+          subject(:merged_yaml) { result.merged_yaml }
+
+          let(:config_content) do
+            YAML.dump(
+              include: { remote: 'https://example.com/sample.yml' },
+              test: { stage: 'test', script: 'echo' }
+            )
+          end
+
+          let(:included_yml) do
+            YAML.dump(
+              another_test: { stage: 'test', script: 'echo 2' }
+            )
+          end
+
+          before do
+            stub_full_request('https://example.com/sample.yml').to_return(body: included_yml)
+          end
+
+          it 'returns expanded yaml config' do
+            expanded_config = YAML.safe_load(merged_yaml, [Symbol])
+            included_config = YAML.safe_load(included_yml, [Symbol])
+
+            expect(expanded_config).to include(*included_config.keys)
+          end
+        end
+      end
+    end
+  end
+end

spec/requests/api/lint_spec.rb

@@ -17,24 +17,53 @@ RSpec.describe API::Lint do
         expect(json_response['status']).to eq('valid')
         expect(json_response['errors']).to eq([])
       end
+
+      it 'outputs expanded yaml content' do
+        post api('/ci/lint'), params: { content: yaml_content, include_merged_yaml: true }
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response).to have_key('merged_yaml')
+      end
     end
 
     context 'with an invalid .gitlab_ci.yml' do
+      context 'with invalid syntax' do
+        let(:yaml_content) { 'invalid content' }
+
         it 'responds with errors about invalid syntax' do
-        post api('/ci/lint'), params: { content: 'invalid content' }
+          post api('/ci/lint'), params: { content: yaml_content }
 
           expect(response).to have_gitlab_http_status(:ok)
           expect(json_response['status']).to eq('invalid')
           expect(json_response['errors']).to eq(['Invalid configuration format'])
         end
 
-      it "responds with errors about invalid configuration" do
-        post api('/ci/lint'), params: { content: '{ image: "ruby:2.7",  services: ["postgres"] }' }
+        it 'outputs expanded yaml content' do
+          post api('/ci/lint'), params: { content: yaml_content, include_merged_yaml: true }
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response).to have_key('merged_yaml')
+        end
+      end
+
+      context 'with invalid configuration' do
+        let(:yaml_content) { '{ image: "ruby:2.7",  services: ["postgres"] }' }
+
+        it 'responds with errors about invalid configuration' do
+          post api('/ci/lint'), params: { content: yaml_content }
 
           expect(response).to have_gitlab_http_status(:ok)
           expect(json_response['status']).to eq('invalid')
           expect(json_response['errors']).to eq(['jobs config should contain at least one visible job'])
         end
+
+        it 'outputs expanded yaml content' do
+          post api('/ci/lint'), params: { content: yaml_content, include_merged_yaml: true }
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response).to have_key('merged_yaml')
+        end
+      end
     end
 
     context 'without the content parameter' do