Commit 0103d5be authored by Kamil Trzciński's avatar Kamil Trzciński

Add config_options|variables to BuildMetadata

These are data columns that store runtime configuration
of build needed to execute it on runner and within pipeline.

The definition of this data is that once used, and when no longer
needed (due to retry capability) they can be freely removed.

They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking
support for json datatype on old enough version).
parent b647ad96
...@@ -8,10 +8,15 @@ module Ci ...@@ -8,10 +8,15 @@ module Ci
include ObjectStorage::BackgroundMove include ObjectStorage::BackgroundMove
include Presentable include Presentable
include Importable include Importable
include IgnorableColumn
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
include Deployable include Deployable
include HasRef include HasRef
BuildArchivedError = Class.new(StandardError)
ignore_column :commands
belongs_to :project, inverse_of: :builds belongs_to :project, inverse_of: :builds
belongs_to :runner belongs_to :runner
belongs_to :trigger_request belongs_to :trigger_request
...@@ -31,7 +36,7 @@ module Ci ...@@ -31,7 +36,7 @@ module Ci
has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
end end
has_one :metadata, class_name: 'Ci::BuildMetadata' has_one :metadata, class_name: 'Ci::BuildMetadata', autosave: true
has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
accepts_nested_attributes_for :runner_session accepts_nested_attributes_for :runner_session
...@@ -273,11 +278,14 @@ module Ci ...@@ -273,11 +278,14 @@ module Ci
# degenerated build is one that cannot be run by Runner # degenerated build is one that cannot be run by Runner
def degenerated? def degenerated?
self.options.nil? self.options.blank?
end end
def degenerate! def degenerate!
self.update!(options: nil, yaml_variables: nil, commands: nil) Build.transaction do
self.update!(options: nil, yaml_variables: nil)
self.metadata&.destroy
end
end end
def archived? def archived?
...@@ -624,11 +632,23 @@ module Ci ...@@ -624,11 +632,23 @@ module Ci
end end
def when def when
read_attribute(:when) || build_attributes_from_config[:when] || 'on_success' read_attribute(:when) || 'on_success'
end
def options
read_metadata_attribute(:options, :config_options, {})
end end
def yaml_variables def yaml_variables
read_attribute(:yaml_variables) || build_attributes_from_config[:yaml_variables] || [] read_metadata_attribute(:yaml_variables, :config_variables, [])
end
def options=(value)
write_metadata_attribute(:options, :config_options, value)
end
def yaml_variables=(value)
write_metadata_attribute(:yaml_variables, :config_variables, value)
end end
def user_variables def user_variables
...@@ -904,8 +924,11 @@ module Ci ...@@ -904,8 +924,11 @@ module Ci
# have the old integer only format. This method returns the retry option # have the old integer only format. This method returns the retry option
# normalized as a hash in 11.5+ format. # normalized as a hash in 11.5+ format.
def normalized_retry def normalized_retry
value = options&.dig(:retry) strong_memoize(:normalized_retry) do
value.is_a?(Integer) ? { max: value } : value.to_h value = options&.dig(:retry)
value = value.is_a?(Integer) ? { max: value } : value.to_h
value.with_indifferent_access
end
end end
def build_attributes_from_config def build_attributes_from_config
...@@ -929,5 +952,20 @@ module Ci ...@@ -929,5 +952,20 @@ module Ci
def project_destroyed? def project_destroyed?
project.pending_delete? project.pending_delete?
end end
def read_metadata_attribute(legacy_key, metadata_key, default_value = nil)
read_attribute(legacy_key) || metadata&.read_attribute(metadata_key) || default_value
end
def write_metadata_attribute(legacy_key, metadata_key, value)
# save to metadata or this model depending on the state of feature flag
if Feature.enabled?(:ci_build_metadata_config)
ensure_metadata.write_attribute(metadata_key, value)
write_attribute(legacy_key, nil)
else
write_attribute(legacy_key, value)
metadata&.write_attribute(metadata_key, nil)
end
end
end end
end end
...@@ -13,8 +13,12 @@ module Ci ...@@ -13,8 +13,12 @@ module Ci
belongs_to :build, class_name: 'Ci::Build' belongs_to :build, class_name: 'Ci::Build'
belongs_to :project belongs_to :project
before_create :set_build_project
validates :build, presence: true validates :build, presence: true
validates :project, presence: true
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
chronic_duration_attr_reader :timeout_human_readable, :timeout chronic_duration_attr_reader :timeout_human_readable, :timeout
...@@ -33,5 +37,11 @@ module Ci ...@@ -33,5 +37,11 @@ module Ci
update(timeout: timeout, timeout_source: timeout_source) update(timeout: timeout, timeout_source: timeout_source)
end end
private
def set_build_project
self.project_id ||= self.build.project_id
end
end end
end end
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Ci module Ci
class RetryBuildService < ::BaseService class RetryBuildService < ::BaseService
CLONE_ACCESSORS = %i[pipeline project ref tag options commands name CLONE_ACCESSORS = %i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request allow_failure stage stage_id stage_idx trigger_request
yaml_variables when environment coverage_regex yaml_variables when environment coverage_regex
description tag_list protected].freeze description tag_list protected].freeze
......
...@@ -13,20 +13,23 @@ ...@@ -13,20 +13,23 @@
%tbody %tbody
- @stages.each do |stage| - @stages.each do |stage|
- @builds.select { |build| build[:stage] == stage }.each do |build| - @builds.select { |build| build[:stage] == stage }.each do |build|
- job = @jobs[build[:name].to_sym]
%tr %tr
%td #{stage.capitalize} Job - #{build[:name]} %td #{stage.capitalize} Job - #{build[:name]}
%td %td
%pre= build[:commands] %pre= job[:before_script].to_a.join('\n')
%pre= job[:script].to_a.join('\n')
%pre= job[:after_script].to_a.join('\n')
%br %br
%b Tag list: %b Tag list:
= build[:tag_list].to_a.join(", ") = build[:tag_list].to_a.join(", ")
%br %br
%b Only policy: %b Only policy:
= @jobs[build[:name].to_sym][:only].to_a.join(", ") = job[:only].to_a.join(", ")
%br %br
%b Except policy: %b Except policy:
= @jobs[build[:name].to_sym][:except].to_a.join(", ") = job[:except].to_a.join(", ")
%br %br
%b Environment: %b Environment:
= build[:environment] = build[:environment]
......
# frozen_string_literal: true
require 'active_record/connection_adapters/abstract_mysql_adapter'
require 'active_record/connection_adapters/mysql/schema_definitions'
# MySQL (5.6) and MariaDB (10.1) are currently supported versions within GitLab,
# Since they do not support native `json` datatype we force to emulate it as `text`
if Gitlab::Database.mysql?
module ActiveRecord
module ConnectionAdapters
class AbstractMysqlAdapter
JSON_DATASIZE = 1.megabyte
NATIVE_DATABASE_TYPES.merge!(
json: { name: "text", limit: JSON_DATASIZE },
jsonb: { name: "text", limit: JSON_DATASIZE }
)
end
module MySQL
module ColumnMethods
# We add `jsonb` helper, as `json` is already defined for `MySQL` since Rails 5
def jsonb(*args, **options)
args.each { |name| column(name, :json, options) }
end
end
end
end
end
end
...@@ -102,14 +102,15 @@ class Gitlab::Seeder::Pipelines ...@@ -102,14 +102,15 @@ class Gitlab::Seeder::Pipelines
[] []
end end
def create_pipeline!(project, ref, commit) def create_pipeline!(project, ref, commit)
project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push) project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
end end
def build_create!(pipeline, opts = {}) def build_create!(pipeline, opts = {})
attributes = job_attributes(pipeline, opts) attributes = job_attributes(pipeline, opts)
.merge(commands: '$ build command')
attributes[:options] ||= {}
attributes[:options][:script] = 'build command'
Ci::Build.create!(attributes).tap do |build| Ci::Build.create!(attributes).tap do |build|
# We need to set build trace and artifacts after saving a build # We need to set build trace and artifacts after saving a build
......
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
add_column :ci_builds_metadata, :config_variables, :jsonb
end
end
...@@ -374,6 +374,8 @@ ActiveRecord::Schema.define(version: 20190103140724) do ...@@ -374,6 +374,8 @@ ActiveRecord::Schema.define(version: 20190103140724) do
t.integer "project_id", null: false t.integer "project_id", null: false
t.integer "timeout" t.integer "timeout"
t.integer "timeout_source", default: 1, null: false t.integer "timeout_source", default: 1, null: false
t.jsonb "config_options"
t.jsonb "config_variables"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree
t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree
end end
......
...@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist ...@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist
suddenly use a different timezone when the system's timezone changes. It also makes it very clear which suddenly use a different timezone when the system's timezone changes. It also makes it very clear which
timezone was used in the first place. timezone was used in the first place.
## Storing JSON in database
The Rails 5 natively supports `JSONB` (binary JSON) column type.
Example migration adding this column:
```ruby
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
end
end
```
On MySQL the `JSON` and `JSONB` is translated to `TEXT 1MB`, as `JSONB` is PostgreSQL only feature.
For above reason you have to use a serializer to provide a translation layer
in order to support PostgreSQL and MySQL seamlessly:
```ruby
class BuildMetadata
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
end
```
## Testing ## Testing
......
...@@ -15,7 +15,6 @@ module Gitlab ...@@ -15,7 +15,6 @@ module Gitlab
def from_commands(job) def from_commands(job)
self.new(:script).tap do |step| self.new(:script).tap do |step|
step.script = job.options[:before_script].to_a + job.options[:script].to_a step.script = job.options[:before_script].to_a + job.options[:script].to_a
step.script = job.commands.split("\n") if step.script.empty?
step.timeout = job.metadata_timeout step.timeout = job.metadata_timeout
step.when = WHEN_ON_SUCCESS step.when = WHEN_ON_SUCCESS
end end
......
...@@ -95,7 +95,7 @@ module Gitlab ...@@ -95,7 +95,7 @@ module Gitlab
helpers :before_script, :script, :stage, :type, :after_script, helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables, :cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage, :retry, :artifacts, :environment, :coverage, :retry,
:parallel :parallel
attributes :script, :tags, :allow_failure, :when, :dependencies, attributes :script, :tags, :allow_failure, :when, :dependencies,
...@@ -121,10 +121,6 @@ module Gitlab ...@@ -121,10 +121,6 @@ module Gitlab
@config.merge(to_hash.compact) @config.merge(to_hash.compact)
end end
def commands
(before_script_value.to_a + script_value.to_a).join("\n")
end
def manual_action? def manual_action?
self.when == 'manual' self.when == 'manual'
end end
...@@ -156,7 +152,6 @@ module Gitlab ...@@ -156,7 +152,6 @@ module Gitlab
{ name: name, { name: name,
before_script: before_script_value, before_script: before_script_value,
script: script_value, script: script_value,
commands: commands,
image: image_value, image: image_value,
services: services_value, services: services_value,
stage: stage_value, stage: stage_value,
......
...@@ -33,7 +33,6 @@ module Gitlab ...@@ -33,7 +33,6 @@ module Gitlab
{ stage_idx: @stages.index(job[:stage]), { stage_idx: @stages.index(job[:stage]),
stage: job[:stage], stage: job[:stage],
commands: job[:commands],
tag_list: job[:tags] || [], tag_list: job[:tags] || [],
name: job[:name].to_s, name: job[:name].to_s,
allow_failure: job[:ignore], allow_failure: job[:ignore],
......
...@@ -148,6 +148,7 @@ excluded_attributes: ...@@ -148,6 +148,7 @@ excluded_attributes:
- :when - :when
- :artifacts_file - :artifacts_file
- :artifacts_metadata - :artifacts_metadata
- :commands
push_event_payload: push_event_payload:
- :event_id - :event_id
project_badges: project_badges:
......
...@@ -150,6 +150,7 @@ module Gitlab ...@@ -150,6 +150,7 @@ module Gitlab
if BUILD_MODELS.include?(@relation_name) if BUILD_MODELS.include?(@relation_name)
@relation_hash.delete('trace') # old export files have trace @relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token') @relation_hash.delete('token')
@relation_hash.delete('commands')
imported_object imported_object
elsif @relation_name == :merge_requests elsif @relation_name == :merge_requests
......
...@@ -115,5 +115,15 @@ module Gitlab ...@@ -115,5 +115,15 @@ module Gitlab
string_or_array.split(',').map(&:strip) string_or_array.split(',').map(&:strip)
end end
def deep_indifferent_access(data)
if data.is_a?(Array)
data.map(&method(:deep_indifferent_access))
elsif data.is_a?(Hash)
data.with_indifferent_access
else
data
end
end
end end
end end
# frozen_string_literal: true
module Serializers
# This serializer exports data as JSON,
# it is designed to be used with interwork compatibility between MySQL and PostgreSQL
# implementations, as used version of MySQL does not support native json type
#
# Secondly, the loader makes the resulting hash to have deep indifferent access
class JSON
class << self
def dump(obj)
# MySQL stores data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
obj = ActiveSupport::JSON.encode(obj)
end
obj
end
def load(data)
return if data.nil?
# On MySQL we store data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
data = ActiveSupport::JSON.decode(data)
end
Gitlab::Utils.deep_indifferent_access(data)
end
end
end
end
...@@ -7,7 +7,6 @@ FactoryBot.define do ...@@ -7,7 +7,6 @@ FactoryBot.define do
stage_idx 0 stage_idx 0
ref 'master' ref 'master'
tag false tag false
commands 'ls -a'
protected false protected false
created_at 'Di 29. Okt 09:50:00 CET 2013' created_at 'Di 29. Okt 09:50:00 CET 2013'
pending pending
...@@ -15,7 +14,8 @@ FactoryBot.define do ...@@ -15,7 +14,8 @@ FactoryBot.define do
options do options do
{ {
image: 'ruby:2.1', image: 'ruby:2.1',
services: ['postgres'] services: ['postgres'],
script: ['ls -a']
} }
end end
...@@ -28,7 +28,6 @@ FactoryBot.define do ...@@ -28,7 +28,6 @@ FactoryBot.define do
pipeline factory: :ci_pipeline pipeline factory: :ci_pipeline
trait :degenerated do trait :degenerated do
commands nil
options nil options nil
yaml_variables nil yaml_variables nil
end end
...@@ -95,33 +94,53 @@ FactoryBot.define do ...@@ -95,33 +94,53 @@ FactoryBot.define do
trait :teardown_environment do trait :teardown_environment do
environment 'staging' environment 'staging'
options environment: { name: 'staging', options do
action: 'stop', {
url: 'http://staging.example.com/$CI_JOB_NAME' } script: %w(ls),
environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
}
end
end end
trait :deploy_to_production do trait :deploy_to_production do
environment 'production' environment 'production'
options environment: { name: 'production', options do
url: 'http://prd.example.com/$CI_JOB_NAME' } {
script: %w(ls),
environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
}
end
end end
trait :start_review_app do trait :start_review_app do
environment 'review/$CI_COMMIT_REF_NAME' environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME', options do
url: 'http://staging.example.com/$CI_JOB_NAME', {
on_stop: 'stop_review_app' } script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
}
end
end end
trait :stop_review_app do trait :stop_review_app do
name 'stop_review_app' name 'stop_review_app'
environment 'review/$CI_COMMIT_REF_NAME' environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME', options do
url: 'http://staging.example.com/$CI_JOB_NAME', {
action: 'stop' } script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
action: 'stop' }
}
end
end end
trait :allowed_to_fail do trait :allowed_to_fail do
...@@ -142,7 +161,13 @@ FactoryBot.define do ...@@ -142,7 +161,13 @@ FactoryBot.define do
trait :schedulable do trait :schedulable do
self.when 'delayed' self.when 'delayed'
options start_in: '1 minute'
options do
{
script: ['ls -a'],
start_in: '1 minute'
}
end
end end
trait :actionable do trait :actionable do
...@@ -265,6 +290,7 @@ FactoryBot.define do ...@@ -265,6 +290,7 @@ FactoryBot.define do
{ {
image: { name: 'ruby:2.1', entrypoint: '/bin/sh' }, image: { name: 'ruby:2.1', entrypoint: '/bin/sh' },
services: ['postgres', { name: 'docker:stable-dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }], services: ['postgres', { name: 'docker:stable-dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }],
script: %w(echo),
after_script: %w(ls date), after_script: %w(ls date),
artifacts: { artifacts: {
name: 'artifacts_file', name: 'artifacts_file',
......
...@@ -5,7 +5,7 @@ describe 'Merge request < User sees mini pipeline graph', :js do ...@@ -5,7 +5,7 @@ describe 'Merge request < User sees mini pipeline graph', :js do
let(:user) { project.creator } let(:user) { project.creator }
let(:merge_request) { create(:merge_request, source_project: project, head_pipeline: pipeline) } let(:merge_request) { create(:merge_request, source_project: project, head_pipeline: pipeline) }
let(:pipeline) { create(:ci_empty_pipeline, project: project, ref: 'master', status: 'running', sha: project.commit.id) } let(:pipeline) { create(:ci_empty_pipeline, project: project, ref: 'master', status: 'running', sha: project.commit.id) }
let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test', commands: 'test') } let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test') }
before do before do
build.run build.run
......
...@@ -272,8 +272,7 @@ describe 'Environments page', :js do ...@@ -272,8 +272,7 @@ describe 'Environments page', :js do
create(:ci_build, :scheduled, create(:ci_build, :scheduled,
pipeline: pipeline, pipeline: pipeline,
name: 'delayed job', name: 'delayed job',
stage: 'test', stage: 'test')
commands: 'test')
end end
let!(:deployment) do let!(:deployment) do
...@@ -304,8 +303,7 @@ describe 'Environments page', :js do ...@@ -304,8 +303,7 @@ describe 'Environments page', :js do
create(:ci_build, :expired_scheduled, create(:ci_build, :expired_scheduled,
pipeline: pipeline, pipeline: pipeline,
name: 'delayed job', name: 'delayed job',
stage: 'test', stage: 'test')
commands: 'test')
end end
it "shows 00:00:00 as the remaining time" do it "shows 00:00:00 as the remaining time" do
......
...@@ -18,7 +18,7 @@ describe 'Pipeline', :js do ...@@ -18,7 +18,7 @@ describe 'Pipeline', :js do
let!(:build_failed) do let!(:build_failed) do
create(:ci_build, :failed, create(:ci_build, :failed,
pipeline: pipeline, stage: 'test', name: 'test', commands: 'test') pipeline: pipeline, stage: 'test', name: 'test')
end end
let!(:build_running) do let!(:build_running) do
......
...@@ -109,8 +109,7 @@ describe 'Pipelines', :js do ...@@ -109,8 +109,7 @@ describe 'Pipelines', :js do
context 'when pipeline is cancelable' do context 'when pipeline is cancelable' do
let!(:build) do let!(:build) do
create(:ci_build, pipeline: pipeline, create(:ci_build, pipeline: pipeline,
stage: 'test', stage: 'test')
commands: 'test')
end end
before do before do
...@@ -140,8 +139,7 @@ describe 'Pipelines', :js do ...@@ -140,8 +139,7 @@ describe 'Pipelines', :js do
context 'when pipeline is retryable' do context 'when pipeline is retryable' do
let!(:build) do let!(:build) do
create(:ci_build, pipeline: pipeline, create(:ci_build, pipeline: pipeline,
stage: 'test', stage: 'test')
commands: 'test')
end end
before do before do
...@@ -202,8 +200,7 @@ describe 'Pipelines', :js do ...@@ -202,8 +200,7 @@ describe 'Pipelines', :js do
create(:ci_build, :manual, create(:ci_build, :manual,
pipeline: pipeline, pipeline: pipeline,
name: 'manual build', name: 'manual build',
stage: 'test', stage: 'test')
commands: 'test')
end end
before do before do
...@@ -237,8 +234,7 @@ describe 'Pipelines', :js do ...@@ -237,8 +234,7 @@ describe 'Pipelines', :js do
create(:ci_build, :scheduled, create(:ci_build, :scheduled,
pipeline: pipeline, pipeline: pipeline,
name: 'delayed job', name: 'delayed job',
stage: 'test', stage: 'test')
commands: 'test')
end end
before do before do
...@@ -262,8 +258,7 @@ describe 'Pipelines', :js do ...@@ -262,8 +258,7 @@ describe 'Pipelines', :js do
create(:ci_build, :expired_scheduled, create(:ci_build, :expired_scheduled,
pipeline: pipeline, pipeline: pipeline,
name: 'delayed job', name: 'delayed job',
stage: 'test', stage: 'test')
commands: 'test')
end end
it "shows 00:00:00 as the remaining time" do it "shows 00:00:00 as the remaining time" do
......
...@@ -14,8 +14,7 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do ...@@ -14,8 +14,7 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do
create(:ci_build, :scheduled, create(:ci_build, :scheduled,
pipeline: pipeline, pipeline: pipeline,
name: 'delayed job', name: 'delayed job',
stage: 'test', stage: 'test')
commands: 'test')
end end
render_views render_views
......
...@@ -5,6 +5,11 @@ require 'spec_helper' ...@@ -5,6 +5,11 @@ require 'spec_helper'
describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do
let(:migration) { described_class.new } let(:migration) { described_class.new }
before do
# This migration was created before we introduced metadata configs
stub_feature_flags(ci_build_metadata_config: false)
end
let!(:internal_pipeline) { create(:ci_pipeline, source: :web) } let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
let(:pipelines) { [internal_pipeline, unknown_pipeline].map(&:id) } let(:pipelines) { [internal_pipeline, unknown_pipeline].map(&:id) }
......
...@@ -18,13 +18,6 @@ describe Gitlab::Ci::Build::Step do ...@@ -18,13 +18,6 @@ describe Gitlab::Ci::Build::Step do
end end
end end
context 'when commands are specified' do
it_behaves_like 'has correct script' do
let(:job) { create(:ci_build, :no_options, commands: "ls -la\ndate") }
let(:script) { ['ls -la', 'date'] }
end
end
context 'when script option is specified' do context 'when script option is specified' do
it_behaves_like 'has correct script' do it_behaves_like 'has correct script' do
let(:job) { create(:ci_build, :no_options, options: { script: ["ls -la\necho aaa", "date"] }) } let(:job) { create(:ci_build, :no_options, options: { script: ["ls -la\necho aaa", "date"] }) }
...@@ -62,7 +55,7 @@ describe Gitlab::Ci::Build::Step do ...@@ -62,7 +55,7 @@ describe Gitlab::Ci::Build::Step do
end end
context 'when after_script is not empty' do context 'when after_script is not empty' do
let(:job) { create(:ci_build, options: { after_script: ['ls -la', 'date'] }) } let(:job) { create(:ci_build, options: { script: ['bash'], after_script: ['ls -la', 'date'] }) }
it 'fabricates an object' do it 'fabricates an object' do
expect(subject.name).to eq(:after_script) expect(subject.name).to eq(:after_script)
......
...@@ -153,7 +153,6 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -153,7 +153,6 @@ describe Gitlab::Ci::Config::Entry::Global do
rspec: { name: :rspec, rspec: { name: :rspec,
script: %w[rspec ls], script: %w[rspec ls],
before_script: %w(ls pwd), before_script: %w(ls pwd),
commands: "ls\npwd\nrspec\nls",
image: { name: 'ruby:2.2' }, image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test', stage: 'test',
...@@ -166,7 +165,6 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -166,7 +165,6 @@ describe Gitlab::Ci::Config::Entry::Global do
spinach: { name: :spinach, spinach: { name: :spinach,
before_script: [], before_script: [],
script: %w[spinach], script: %w[spinach],
commands: 'spinach',
image: { name: 'ruby:2.2' }, image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test', stage: 'test',
......
...@@ -255,7 +255,6 @@ describe Gitlab::Ci::Config::Entry::Job do ...@@ -255,7 +255,6 @@ describe Gitlab::Ci::Config::Entry::Job do
.to eq(name: :rspec, .to eq(name: :rspec,
before_script: %w[ls pwd], before_script: %w[ls pwd],
script: %w[rspec], script: %w[rspec],
commands: "ls\npwd\nrspec",
stage: 'test', stage: 'test',
ignore: false, ignore: false,
after_script: %w[cleanup], after_script: %w[cleanup],
...@@ -264,16 +263,6 @@ describe Gitlab::Ci::Config::Entry::Job do ...@@ -264,16 +263,6 @@ describe Gitlab::Ci::Config::Entry::Job do
end end
end end
end end
describe '#commands' do
let(:config) do
{ before_script: %w[ls pwd], script: 'rspec' }
end
it 'returns a string of commands concatenated with new line character' do
expect(entry.commands).to eq "ls\npwd\nrspec"
end
end
end end
describe '#manual_action?' do describe '#manual_action?' do
......
...@@ -65,14 +65,12 @@ describe Gitlab::Ci::Config::Entry::Jobs do ...@@ -65,14 +65,12 @@ describe Gitlab::Ci::Config::Entry::Jobs do
expect(entry.value).to eq( expect(entry.value).to eq(
rspec: { name: :rspec, rspec: { name: :rspec,
script: %w[rspec], script: %w[rspec],
commands: 'rspec',
ignore: false, ignore: false,
stage: 'test', stage: 'test',
only: { refs: %w[branches tags] }, only: { refs: %w[branches tags] },
except: {} }, except: {} },
spinach: { name: :spinach, spinach: { name: :spinach,
script: %w[spinach], script: %w[spinach],
commands: 'spinach',
ignore: false, ignore: false,
stage: 'test', stage: 'test',
only: { refs: %w[branches tags] }, only: { refs: %w[branches tags] },
......
...@@ -6,8 +6,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do ...@@ -6,8 +6,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
let(:attributes) do let(:attributes) do
{ name: 'rspec', { name: 'rspec',
ref: 'master', ref: 'master' }
commands: 'rspec' }
end end
subject do subject do
...@@ -18,7 +17,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do ...@@ -18,7 +17,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
it 'returns hash attributes of a build' do it 'returns hash attributes of a build' do
expect(subject.attributes).to be_a Hash expect(subject.attributes).to be_a Hash
expect(subject.attributes) expect(subject.attributes)
.to include(:name, :project, :ref, :commands) .to include(:name, :project, :ref)
end end
end end
......
...@@ -21,7 +21,6 @@ module Gitlab ...@@ -21,7 +21,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -155,7 +154,6 @@ module Gitlab ...@@ -155,7 +154,6 @@ module Gitlab
builds: builds:
[{ stage_idx: 1, [{ stage_idx: 1,
stage: "test", stage: "test",
commands: "rspec",
tag_list: [], tag_list: [],
name: "rspec", name: "rspec",
allow_failure: false, allow_failure: false,
...@@ -171,7 +169,6 @@ module Gitlab ...@@ -171,7 +169,6 @@ module Gitlab
builds: builds:
[{ stage_idx: 2, [{ stage_idx: 2,
stage: "deploy", stage: "deploy",
commands: "cap prod",
tag_list: [], tag_list: [],
name: "prod", name: "prod",
allow_failure: false, allow_failure: false,
...@@ -271,7 +268,7 @@ module Gitlab ...@@ -271,7 +268,7 @@ module Gitlab
end end
it "return commands with scripts concencaced" do it "return commands with scripts concencaced" do
expect(subject[:commands]).to eq("global script\nscript") expect(subject[:options][:before_script]).to eq(["global script"])
end end
end end
...@@ -284,7 +281,7 @@ module Gitlab ...@@ -284,7 +281,7 @@ module Gitlab
end end
it "return commands with scripts concencaced" do it "return commands with scripts concencaced" do
expect(subject[:commands]).to eq("local script\nscript") expect(subject[:options][:before_script]).to eq(["local script"])
end end
end end
end end
...@@ -297,7 +294,7 @@ module Gitlab ...@@ -297,7 +294,7 @@ module Gitlab
end end
it "return commands with scripts concencaced" do it "return commands with scripts concencaced" do
expect(subject[:commands]).to eq("script") expect(subject[:options][:script]).to eq(["script"])
end end
end end
...@@ -347,7 +344,6 @@ module Gitlab ...@@ -347,7 +344,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -382,7 +378,6 @@ module Gitlab ...@@ -382,7 +378,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -415,7 +410,6 @@ module Gitlab ...@@ -415,7 +410,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -444,7 +438,6 @@ module Gitlab ...@@ -444,7 +438,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -596,7 +589,7 @@ module Gitlab ...@@ -596,7 +589,7 @@ module Gitlab
it 'correctly extends rspec job' do it 'correctly extends rspec job' do
expect(config_processor.builds).to be_one expect(config_processor.builds).to be_one
expect(subject.dig(:commands)).to eq 'test' expect(subject.dig(:options, :script)).to eq %w(test)
expect(subject.dig(:options, :image, :name)).to eq 'ruby:alpine' expect(subject.dig(:options, :image, :name)).to eq 'ruby:alpine'
end end
end end
...@@ -622,7 +615,8 @@ module Gitlab ...@@ -622,7 +615,8 @@ module Gitlab
it 'correctly extends rspec job' do it 'correctly extends rspec job' do
expect(config_processor.builds).to be_one expect(config_processor.builds).to be_one
expect(subject.dig(:commands)).to eq "bundle install\nrspec" expect(subject.dig(:options, :before_script)).to eq ["bundle install"]
expect(subject.dig(:options, :script)).to eq %w(rspec)
expect(subject.dig(:options, :image, :name)).to eq 'image:test' expect(subject.dig(:options, :image, :name)).to eq 'image:test'
expect(subject.dig(:when)).to eq 'always' expect(subject.dig(:when)).to eq 'always'
end end
...@@ -769,7 +763,6 @@ module Gitlab ...@@ -769,7 +763,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "rspec", name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -983,7 +976,6 @@ module Gitlab ...@@ -983,7 +976,6 @@ module Gitlab
stage: "test", stage: "test",
stage_idx: 1, stage_idx: 1,
name: "normal_job", name: "normal_job",
commands: "test",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -1031,7 +1023,6 @@ module Gitlab ...@@ -1031,7 +1023,6 @@ module Gitlab
stage: "build", stage: "build",
stage_idx: 0, stage_idx: 0,
name: "job1", name: "job1",
commands: "execute-script-for-job",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
...@@ -1046,7 +1037,6 @@ module Gitlab ...@@ -1046,7 +1037,6 @@ module Gitlab
stage: "build", stage: "build",
stage_idx: 0, stage_idx: 0,
name: "job2", name: "job2",
commands: "execute-script-for-job",
coverage_regex: nil, coverage_regex: nil,
tag_list: [], tag_list: [],
options: { options: {
......
...@@ -197,4 +197,20 @@ describe Gitlab::Utils do ...@@ -197,4 +197,20 @@ describe Gitlab::Utils do
end end
end end
end end
describe '.deep_indifferent_access' do
let(:hash) do
{ "variables" => [{ "key" => "VAR1", "value" => "VALUE2" }] }
end
subject { described_class.deep_indifferent_access(hash) }
it 'allows to access hash keys with symbols' do
expect(subject[:variables]).to be_a(Array)
end
it 'allows to access array keys with symbols' do
expect(subject[:variables].first[:key]).to eq('VAR1')
end
end
end end
require 'fast_spec_helper'
describe Serializers::JSON do
describe '.dump' do
let(:obj) { { key: "value" } }
subject { described_class.dump(obj) }
context 'when MySQL is used' do
before do
allow(Gitlab::Database).to receive(:adapter_name) { 'mysql2' }
end
it 'encodes as string' do
is_expected.to eq('{"key":"value"}')
end
end
context 'when PostgreSQL is used' do
before do
allow(Gitlab::Database).to receive(:adapter_name) { 'postgresql' }
end
it 'returns a hash' do
is_expected.to eq(obj)
end
end
end
describe '.load' do
let(:data_string) { '{"key":"value","variables":[{"key":"VAR1","value":"VALUE1"}]}' }
let(:data_hash) { JSON.parse(data_string) }
shared_examples 'having consistent accessor' do
it 'allows to access with symbols' do
expect(subject[:key]).to eq('value')
expect(subject[:variables].first[:key]).to eq('VAR1')
end
it 'allows to access with strings' do
expect(subject["key"]).to eq('value')
expect(subject["variables"].first["key"]).to eq('VAR1')
end
end
context 'when MySQL is used' do
before do
allow(Gitlab::Database).to receive(:adapter_name) { 'mysql2' }
end
context 'when loading a string' do
subject { described_class.load(data_string) }
it 'decodes a string' do
is_expected.to be_a(Hash)
end
it_behaves_like 'having consistent accessor'
end
context 'when loading a different type' do
subject { described_class.load({ key: 'hash' }) }
it 'raises an exception' do
expect { subject }.to raise_error(TypeError)
end
end
context 'when loading a nil' do
subject { described_class.load(nil) }
it 'returns nil' do
is_expected.to be_nil
end
end
end
context 'when PostgreSQL is used' do
before do
allow(Gitlab::Database).to receive(:adapter_name) { 'postgresql' }
end
context 'when loading a hash' do
subject { described_class.load(data_hash) }
it 'decodes a string' do
is_expected.to be_a(Hash)
end
it_behaves_like 'having consistent accessor'
end
context 'when loading a nil' do
subject { described_class.load(nil) }
it 'returns nil' do
is_expected.to be_nil
end
end
end
end
end
...@@ -90,6 +90,13 @@ describe DeleteInconsistentInternalIdRecords, :migration do ...@@ -90,6 +90,13 @@ describe DeleteInconsistentInternalIdRecords, :migration do
context 'for ci_pipelines' do context 'for ci_pipelines' do
let(:scope) { :ci_pipeline } let(:scope) { :ci_pipeline }
let(:create_models) do
create_list(:ci_empty_pipeline, 3, project: project1)
create_list(:ci_empty_pipeline, 3, project: project2)
create_list(:ci_empty_pipeline, 3, project: project3)
end
it_behaves_like 'deleting inconsistent internal_id records' it_behaves_like 'deleting inconsistent internal_id records'
end end
......
...@@ -13,12 +13,12 @@ describe Ci::BuildMetadata do ...@@ -13,12 +13,12 @@ describe Ci::BuildMetadata do
end end
let(:build) { create(:ci_build, pipeline: pipeline) } let(:build) { create(:ci_build, pipeline: pipeline) }
let(:build_metadata) { build.metadata } let(:metadata) { build.metadata }
it_behaves_like 'having unique enum values' it_behaves_like 'having unique enum values'
describe '#update_timeout_state' do describe '#update_timeout_state' do
subject { build_metadata } subject { metadata }
context 'when runner is not assigned to the job' do context 'when runner is not assigned to the job' do
it "doesn't change timeout value" do it "doesn't change timeout value" do
......
This diff is collapsed.
...@@ -287,7 +287,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do ...@@ -287,7 +287,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
let(:runner) { create(:ci_runner, :project, projects: [project]) } let(:runner) { create(:ci_runner, :project, projects: [project]) }
let(:job) do let(:job) do
create(:ci_build, :artifacts, :extended_options, create(:ci_build, :artifacts, :extended_options,
pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate") pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0)
end end
describe 'POST /api/v4/jobs/request' do describe 'POST /api/v4/jobs/request' do
...@@ -422,7 +422,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do ...@@ -422,7 +422,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
let(:expected_steps) do let(:expected_steps) do
[{ 'name' => 'script', [{ 'name' => 'script',
'script' => %w(ls date), 'script' => %w(echo),
'timeout' => job.metadata_timeout, 'timeout' => job.metadata_timeout,
'when' => 'on_success', 'when' => 'on_success',
'allow_failure' => false }, 'allow_failure' => false },
...@@ -588,7 +588,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do ...@@ -588,7 +588,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
let!(:test_job) do let!(:test_job) do
create(:ci_build, pipeline: pipeline, token: 'test-job-token', name: 'deploy', create(:ci_build, pipeline: pipeline, token: 'test-job-token', name: 'deploy',
stage: 'deploy', stage_idx: 1, stage: 'deploy', stage_idx: 1,
options: { dependencies: [job2.name] }) options: { script: ['bash'], dependencies: [job2.name] })
end end
before do before do
...@@ -612,7 +612,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do ...@@ -612,7 +612,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
let!(:empty_dependencies_job) do let!(:empty_dependencies_job) do
create(:ci_build, pipeline: pipeline, token: 'test-job-token', name: 'empty_dependencies_job', create(:ci_build, pipeline: pipeline, token: 'test-job-token', name: 'empty_dependencies_job',
stage: 'deploy', stage_idx: 1, stage: 'deploy', stage_idx: 1,
options: { dependencies: [] }) options: { script: ['bash'], dependencies: [] })
end end
before do before do
......
...@@ -671,9 +671,9 @@ describe Ci::ProcessPipelineService, '#execute' do ...@@ -671,9 +671,9 @@ describe Ci::ProcessPipelineService, '#execute' do
context 'when builds with auto-retries are configured' do context 'when builds with auto-retries are configured' do
before do before do
create_build('build:1', stage_idx: 0, user: user, options: { retry: { max: 2 } }) create_build('build:1', stage_idx: 0, user: user, options: { script: 'aa', retry: 2 })
create_build('test:1', stage_idx: 1, user: user, when: :on_failure) create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
create_build('test:2', stage_idx: 1, user: user, options: { retry: { max: 1 } }) create_build('test:2', stage_idx: 1, user: user, options: { script: 'aa', retry: 1 })
end end
it 'automatically retries builds in a valid order' do it 'automatically retries builds in a valid order' do
...@@ -770,7 +770,7 @@ describe Ci::ProcessPipelineService, '#execute' do ...@@ -770,7 +770,7 @@ describe Ci::ProcessPipelineService, '#execute' do
end end
def delayed_options def delayed_options
{ when: 'delayed', options: { start_in: '1 minute' } } { when: 'delayed', options: { script: %w(echo), start_in: '1 minute' } }
end end
def unschedule def unschedule
......
...@@ -460,7 +460,12 @@ module Ci ...@@ -460,7 +460,12 @@ module Ci
end end
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) } let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) }
let!(:pending_job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['test'] } ) }
let!(:pending_job) do
create(:ci_build, :pending,
pipeline: pipeline, stage_idx: 1,
options: { script: ["bash"], dependencies: ['test'] })
end
subject { execute(specific_runner) } subject { execute(specific_runner) }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment