Merge remote-tracking branch 'origin/master' into zj-job-view-goes-real-time

Gemfile

@@ -370,3 +370,7 @@ gem 'sys-filesystem', '~> 1.1.6'
 gem 'gitaly', '~> 0.7.0'
 
 gem 'toml-rb', '~> 0.3.15', require: false
+
+# Feature toggles
+gem 'flipper', '~> 0.10.2'
+gem 'flipper-active_record', '~> 0.10.2'

Gemfile.lock

@@ -206,6 +206,10 @@ GEM
       path_expander (~> 1.0)
       ruby_parser (~> 3.0)
       sexp_processor (~> 4.0)
+    flipper (0.10.2)
+    flipper-active_record (0.10.2)
+      activerecord (>= 3.2, < 6)
+      flipper (~> 0.10.2)
     flowdock (0.7.1)
       httparty (~> 0.7)
       multi_json
@@ -907,6 +911,8 @@ DEPENDENCIES
   faraday (~> 0.11.0)
   ffaker (~> 2.4)
   flay (~> 2.8.0)
+  flipper (~> 0.10.2)
+  flipper-active_record (~> 0.10.2)
   fog-aws (~> 0.9)
   fog-core (~> 1.44)
   fog-google (~> 0.5)

app/controllers/projects/pipelines_controller.rb

@@ -58,7 +58,7 @@ class Projects::PipelinesController < Projects::ApplicationController
   def create
     @pipeline = Ci::CreatePipelineService
       .new(project, current_user, create_params)
-      .execute(ignore_skip_ci: true, save_on_errors: false)
+      .execute(:web, ignore_skip_ci: true, save_on_errors: false)
 
     if @pipeline.persisted?
       redirect_to namespace_project_pipeline_path(project.namespace, project, @pipeline)

app/models/ci/pipeline.rb

@@ -30,6 +30,7 @@ module Ci
 
     delegate :id, to: :project, prefix: true
 
+    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
     validates :sha, presence: { unless: :importing? }
     validates :ref, presence: { unless: :importing? }
     validates :status, presence: { unless: :importing? }
@@ -37,6 +38,16 @@ module Ci
 
     after_create :keep_around_commits, unless: :importing?
 
+    enum source: {
+      unknown: nil,
+      push: 1,
+      web: 2,
+      trigger: 3,
+      schedule: 4,
+      api: 5,
+      external: 6
+    }
+
     state_machine :status, initial: :created do
       event :enqueue do
         transition created: :pending
@@ -269,10 +280,6 @@ module Ci
       commit.sha == sha
     end
 
-    def triggered?
-      trigger_requests.any?
-    end
-
     def retried
       @retried ||= (statuses.order(id: :desc) - statuses.latest)
     end

app/models/project.rb

@@ -1061,11 +1061,6 @@ class Project < ActiveRecord::Base
     pipelines.order(id: :desc).find_by(sha: sha, ref: ref)
   end
 
-  def ensure_pipeline(ref, sha, current_user = nil)
-    pipeline_for(ref, sha) ||
-      pipelines.create(sha: sha, ref: ref, user: current_user)
-  end
-
   def enable_ci
     project_feature.update_attribute(:builds_access_level, ProjectFeature::ENABLED)
   end

app/serializers/pipeline_entity.rb

@@ -5,6 +5,7 @@ class PipelineEntity < Grape::Entity
   expose :user, using: UserEntity
   expose :active?, as: :active
   expose :coverage
+  expose :source
 
   expose :created_at, :updated_at
 
@@ -17,7 +18,6 @@ class PipelineEntity < Grape::Entity
 
   expose :flags do
     expose :latest?, as: :latest
-    expose :triggered?, as: :triggered
     expose :stuck?, as: :stuck
     expose :has_yaml_errors?, as: :yaml_errors
     expose :can_retry?, as: :retryable

app/services/ci/create_pipeline_service.rb

@@ -2,8 +2,9 @@ module Ci
   class CreatePipelineService < BaseService
     attr_reader :pipeline
 
-    def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil)
+    def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil)
       @pipeline = Ci::Pipeline.new(
+        source: source,
         project: project,
         ref: ref,
         sha: sha,

app/services/ci/create_trigger_request_service.rb

@@ -4,7 +4,7 @@ module Ci
       trigger_request = trigger.trigger_requests.create(variables: variables)
 
       pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: ref).
-        execute(ignore_skip_ci: true, trigger_request: trigger_request)
+        execute(:trigger, ignore_skip_ci: true, trigger_request: trigger_request)
 
       trigger_request if pipeline.persisted?
     end

app/services/git_push_service.rb

@@ -106,7 +106,7 @@ class GitPushService < BaseService
     EventCreateService.new.push(@project, current_user, build_push_data)
     @project.execute_hooks(build_push_data.dup, :push_hooks)
     @project.execute_services(build_push_data.dup, :push_hooks)
-    Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute
+    Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute(:push)
 
     if push_remove_branch?
       AfterBranchDeleteService

app/services/git_tag_push_service.rb

@@ -11,7 +11,7 @@ class GitTagPushService < BaseService
     SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks)
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
-    Ci::CreatePipelineService.new(project, current_user, @push_data).execute
+    Ci::CreatePipelineService.new(project, current_user, @push_data).execute(:push)
     ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 
     true

app/workers/pipeline_schedule_worker.rb

@@ -14,7 +14,7 @@ class PipelineScheduleWorker
         Ci::CreatePipelineService.new(schedule.project,
                                       schedule.owner,
                                       ref: schedule.ref)
-          .execute(save_on_errors: false, schedule: schedule)
+          .execute(:schedule, save_on_errors: false, schedule: schedule)
       rescue => e
         Rails.logger.error "#{schedule.id}: Failed to create a scheduled pipeline: #{e.message}"
       ensure

changelogs/unreleased/dm-gitmodules-parsing.yml

+---
+title: Make .gitmodules parsing more resilient to syntax errors
+merge_request:
+author:

changelogs/unreleased/feature-flags-flipper.yml

+---
+title: Add feature toggles and API endpoints for admins
+merge_request: 11747
+author:

changelogs/unreleased/introduce-source-to-pipelines.yml

+---
+title: Introduce source to Pipeline entity
+merge_request:
+author:

db/fixtures/development/14_pipelines.rb

@@ -98,7 +98,7 @@ class Gitlab::Seeder::Pipelines
 
 
   def create_pipeline!(project, ref, commit)
-    project.pipelines.create(sha: commit.id, ref: ref)
+    project.pipelines.create(sha: commit.id, ref: ref, source: :push)
   end
 
   def build_create!(pipeline, opts = {})

db/fixtures/development/17_cycle_analytics.rb

@@ -190,7 +190,7 @@ class Gitlab::Seeder::CycleAnalytics
       service = Ci::CreatePipelineService.new(merge_request.project,
                                               @user,
                                               ref: "refs/heads/#{merge_request.source_branch}")
-      pipeline = service.execute(ignore_skip_ci: true, save_on_errors: false)
+      pipeline = service.execute(:push, ignore_skip_ci: true, save_on_errors: false)
 
       pipeline.run!
       Timecop.travel rand(1..6).hours.from_now

db/migrate/20170524125940_add_source_to_ci_pipeline.rb

+class AddSourceToCiPipeline < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_pipelines, :source, :integer
+  end
+end

db/migrate/20170525174156_create_feature_tables.rb

+class CreateFeatureTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def self.up
+    create_table :features do |t|
+      t.string :key, null: false
+      t.timestamps null: false
+    end
+    add_index :features, :key, unique: true
+
+    create_table :feature_gates do |t|
+      t.string :feature_key, null: false
+      t.string :key, null: false
+      t.string :value
+      t.timestamps null: false
+    end
+    add_index :feature_gates, [:feature_key, :key, :value], unique: true
+  end
+
+  def self.down
+    drop_table :feature_gates
+    drop_table :features
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170523091700) do
+ActiveRecord::Schema.define(version: 20170525174156) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -283,6 +283,7 @@ ActiveRecord::Schema.define(version: 20170523091700) do
     t.integer "lock_version"
     t.integer "auto_canceled_by_id"
     t.integer "pipeline_schedule_id"
+    t.integer "source"
   end
 
   add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
@@ -440,6 +441,24 @@ ActiveRecord::Schema.define(version: 20170523091700) do
   add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
   add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
 
+  create_table "feature_gates", force: :cascade do |t|
+    t.string "feature_key", null: false
+    t.string "key", null: false
+    t.string "value"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "feature_gates", ["feature_key", "key", "value"], name: "index_feature_gates_on_feature_key_and_key_and_value", unique: true, using: :btree
+
+  create_table "features", force: :cascade do |t|
+    t.string "key", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "features", ["key"], name: "index_features_on_key", unique: true, using: :btree
+
   create_table "forked_project_links", force: :cascade do |t|
     t.integer "forked_to_project_id", null: false
     t.integer "forked_from_project_id", null: false
@@ -1473,4 +1492,4 @@ ActiveRecord::Schema.define(version: 20170523091700) do
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
-end
+end
\ No newline at end of file

doc/api/features.md

+# Features API
+
+All methods require administrator authorization.
+
+Notice that currently the API only supports boolean and percentage-of-time gate
+values.
+
+## List all features
+
+Get a list of all persisted features, with its gate values.
+
+```
+GET /features
+```
+
+```bash
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/features
+```
+
+Example response:
+
+```json
+[
+  {
+    "name": "experimental_feature",
+    "state": "off",
+    "gates": [
+      {
+        "key": "boolean",
+        "value": false
+      }
+    ]
+  },
+  {
+    "name": "new_library",
+    "state": "on",
+    "gates": [
+      {
+        "key": "boolean",
+        "value": true
+      }
+    ]
+  }
+]
+```
+
+## Set or create a feature
+
+Set a feature's gate value. If a feature with the given name doesn't exist yet
+it will be created. The value can be a boolean, or an integer to indicate
+percentage of time.
+
+```
+POST /features/:name
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `name` | string | yes | Name of the feature to create or update |
+| `value` | integer/string | yes | `true` or `false` to enable/disable, or an integer for percentage of time |
+
+```bash
+curl --data "value=30" --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/features/new_library
+```
+
+Example response:
+
+```json
+{
+  "name": "new_library",
+  "state": "conditional",
+  "gates": [
+    {
+      "key": "boolean",
+      "value": false
+    },
+    {
+      "key": "percentage_of_time",
+      "value": 30
+    }
+  ]
+}
+```

doc/development/README.md

@@ -42,6 +42,7 @@
 - [Sidekiq debugging](sidekiq_debugging.md)
 - [Object state models](object_state_models.md)
 - [Building a package for testing purposes](build_test_package.md)
+- [Manage feature flags](feature_flags.md)
 
 ## Databases
 

doc/development/feature_flags.md

+# Manage feature flags
+
+Starting from GitLab 9.3 we support feature flags via
+[Flipper](https://github.com/jnunemaker/flipper/). You should use the `Feature`
+class (defined in `lib/feature.rb`) in your code to get, set and list feature
+flags. During runtime you can set the values for the gates via the
+[admin API](../api/features.md).

doc/university/README.md

@@ -65,6 +65,7 @@ The curriculum is composed of GitLab videos, screencasts, presentations, project
 1. [Using Innersourcing to Improve Collaboration](https://about.gitlab.com/2014/09/05/innersourcing-using-the-open-source-workflow-to-improve-collaboration-within-an-organization/)
 1. [The Software Development Market and GitLab - Video](https://www.youtube.com/watch?v=sXlhgPK1NTY&list=PLFGfElNsQthbQu_IWlNOxul0TbS_2JH-e&index=6) - [Slides](https://docs.google.com/presentation/d/1vCU-NbZWz8NTNK8Vu3y4zGMAHb5DpC8PE5mHtw1PWfI/edit)
 1. [The GitLab Book Club](bookclub/index.md)
+1. [GitLab Resources](https://about.gitlab.com/resources/)
 
 #### 1.7 Community and Support
 

features/steps/project/pages.rb

@@ -35,7 +35,7 @@ class Spinach::Features::ProjectPages < Spinach::FeatureSteps
   end
 
   step 'pages are deployed' do
-    pipeline = @project.ensure_pipeline('HEAD', @project.commit('HEAD').sha)
+    pipeline = @project.pipelines.create(ref: 'HEAD', sha: @project.commit('HEAD').sha)
     build = build(:ci_build,
                   project: @project,
                   pipeline: pipeline,

lib/api/api.rb

@@ -94,6 +94,7 @@ module API
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
+    mount ::API::Features
     mount ::API::Files
     mount ::API::Groups
     mount ::API::Internal

lib/api/commit_statuses.rb

@@ -68,7 +68,14 @@ module API
 
         name = params[:name] || params[:context] || 'default'
 
-        pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
+        pipeline = @project.pipeline_for(ref, commit.sha)
+        unless pipeline
+          pipeline = @project.pipelines.create!(
+            source: :external,
+            sha: commit.sha,
+            ref: ref,
+            user: current_user)
+        end
 
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
           project: @project,

lib/api/entities.rb

@@ -753,6 +753,28 @@ module API
       expose :impersonation
     end
 
+    class FeatureGate < Grape::Entity
+      expose :key
+      expose :value
+    end
+
+    class Feature < Grape::Entity
+      expose :name
+      expose :state
+      expose :gates, using: FeatureGate do |model|
+        model.gates.map do |gate|
+          value = model.gate_values[gate.key]
+
+          # By default all gate values are populated. Only show relevant ones.
+          if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+            next
+          end
+
+          { key: gate.key, value: value }
+        end.compact
+      end
+    end
+
     module JobRequest
       class JobInfo < Grape::Entity
         expose :name, :stage

lib/api/features.rb

+module API
+  class Features < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :features do
+      desc 'Get a list of all features' do
+        success Entities::Feature
+      end
+      get do
+        features = Feature.all
+
+        present features, with: Entities::Feature, current_user: current_user
+      end
+
+      desc 'Set the gate value for the given feature' do
+        success Entities::Feature
+      end
+      params do
+        requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+      end
+      post ':name' do
+        feature = Feature.get(params[:name])
+
+        if %w(0 false).include?(params[:value])
+          feature.disable
+        elsif params[:value] == 'true'
+          feature.enable
+        else
+          feature.enable_percentage_of_time(params[:value].to_i)
+        end
+
+        present feature, with: Entities::Feature, current_user: current_user
+      end
+    end
+  end
+end

lib/api/pipelines.rb

@@ -47,7 +47,7 @@ module API
         new_pipeline = Ci::CreatePipelineService.new(user_project,
                                                      current_user,
                                                      declared_params(include_missing: false))
-                           .execute(ignore_skip_ci: true, save_on_errors: false)
+                           .execute(:api, ignore_skip_ci: true, save_on_errors: false)
         if new_pipeline.persisted?
           present new_pipeline, with: Entities::Pipeline
         else

lib/feature.rb

+require 'flipper/adapters/active_record'
+
+class Feature
+  # Classes to override flipper table names
+  class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    # Using `self.table_name` won't work. ActiveRecord bug?
+    superclass.table_name = 'features'
+  end
+
+  class FlipperGate < Flipper::Adapters::ActiveRecord::Gate
+    superclass.table_name = 'feature_gates'
+  end
+
+  class << self
+    def all
+      flipper.features.to_a
+    end
+
+    def get(key)
+      flipper.feature(key)
+    end
+
+    def persisted?(feature)
+      # Flipper creates on-memory features when asked for a not-yet-created one.
+      # If we want to check if a feature has been actually set, we look for it
+      # on the persisted features list.
+      all.map(&:name).include?(feature.name)
+    end
+
+    private
+
+    def flipper
+      @flipper ||= begin
+        adapter = Flipper::Adapters::ActiveRecord.new(
+          feature_class: FlipperFeature, gate_class: FlipperGate)
+
+        Flipper.new(adapter)
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -1006,31 +1006,39 @@ module Gitlab
       # Parses the contents of a .gitmodules file and returns a hash of
       # submodule information.
       def parse_gitmodules(commit, content)
-        results = {}
+        modules = {}
 
-        current = ""
-        content.split("\n").each do |txt|
-          if txt =~ /^\s*\[/
-            current = txt.match(/(?<=").*(?=")/)[0]
-            results[current] = {}
-          else
-            next unless results[current]
-            match_data = txt.match(/(\w+)\s*=\s*(.*)/)
-            next unless match_data
-            target = match_data[2].chomp
-            results[current][match_data[1]] = target
+        name = nil
+        content.each_line do |line|
+          case line.strip
+          when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header
+            name = $~[:name]
+            modules[name] = {}
+          when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair
+            key = $~[:key]
+            value = $~[:value].chomp
+
+            next unless name && modules[name]
+
+            modules[name][key] = value
 
-            if match_data[1] == "path"
+            if key == 'path'
               begin
-                results[current]["id"] = blob_content(commit, target)
+                modules[name]['id'] = blob_content(commit, value)
               rescue InvalidBlobName
-                results.delete(current)
+                # The current entry is invalid
+                modules.delete(name)
+                name = nil
               end
             end
+          when /\A#/ # Comment
+            next
+          else # Invalid line
+            name = nil
           end
         end
 
-        results
+        modules
       end
 
       # Returns true if +commit+ introduced changes to +path+, using commit
@@ -1086,7 +1094,12 @@ module Gitlab
           elsif tmp_entry.nil?
             return nil
           else
-            tmp_entry = rugged.lookup(tmp_entry[:oid])
+            begin
+              tmp_entry = rugged.lookup(tmp_entry[:oid])
+            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+              return nil
+            end
+
             return nil unless tmp_entry.type == :tree
             tmp_entry = tmp_entry[dir]
           end

lib/gitlab/gitaly_client.rb

@@ -2,6 +2,12 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    module MigrationStatus
+      DISABLED = 1
+      OPT_IN = 2
+      OPT_OUT = 3
+    end
+
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
 
     MUTEX = Mutex.new
@@ -46,8 +52,20 @@ module Gitlab
       Gitlab.config.gitaly.enabled
     end
 
-    def self.feature_enabled?(feature)
-      enabled? && ENV["GITALY_#{feature.upcase}"] == '1'
+    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+      return false if !enabled? || status == MigrationStatus::DISABLED
+
+      feature = Feature.get("gitaly_#{feature}")
+
+      # If the feature hasn't been set, turn it on if it's opt-out
+      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+
+      if feature.percentage_of_time_value > 0
+        # Probabilistically enable this feature
+        return Random.rand() * 100 < feature.percentage_of_time_value
+      end
+
+      feature.enabled?
     end
 
     def self.migrate(feature)

spec/factories/ci/pipelines.rb

 FactoryGirl.define do
   factory :ci_empty_pipeline, class: Ci::Pipeline do
+    source :push
     ref 'master'
     sha '97de212e80737a608d939f648d959671fb0a0142'
     status 'pending'

spec/features/projects/pipelines/pipelines_spec.rb

@@ -442,6 +442,8 @@ describe 'Pipelines', :feature, :js do
           it 'creates a new pipeline' do
             expect { click_on 'Create pipeline' }
               .to change { Ci::Pipeline.count }.by(1)
+
+            expect(Ci::Pipeline.last).to be_web
           end
         end
 

spec/lib/feature_spec.rb

+require 'spec_helper'
+
+describe Feature, lib: true do
+  describe '.get' do
+    let(:feature) { double(:feature) }
+    let(:key) { 'my_feature' }
+
+    it 'returns the Flipper feature' do
+      expect_any_instance_of(Flipper::DSL).to receive(:feature).with(key).
+        and_return(feature)
+
+      expect(described_class.get(key)).to be(feature)
+    end
+  end
+
+  describe '.all' do
+    let(:features) { Set.new }
+
+    it 'returns the Flipper features as an array' do
+      expect_any_instance_of(Flipper::DSL).to receive(:features).
+        and_return(features)
+
+      expect(described_class.all).to eq(features.to_a)
+    end
+  end
+end

spec/lib/gitlab/git/repository_spec.rb

@@ -381,6 +381,19 @@ describe Gitlab::Git::Repository, seed_helper: true do
           }
         ])
       end
+
+      it 'should not break on invalid syntax' do
+        allow(repository).to receive(:blob_content).and_return(<<-GITMODULES.strip_heredoc)
+          [submodule "six"]
+          path = six
+          url = git://github.com/randx/six.git
+
+          [submodule]
+          foo = bar
+        GITMODULES
+
+        expect(submodules).to have_key('six')
+      end
     end
 
     context 'where repo doesn\'t have submodules' do

spec/lib/gitlab/gitaly_client_spec.rb

 require 'spec_helper'
 
-describe Gitlab::GitalyClient, lib: true do
+# We stub Gitaly in `spec/support/gitaly.rb` for other tests. We don't want
+# those stubs while testing the GitalyClient itself.
+describe Gitlab::GitalyClient, lib: true, skip_gitaly_mock: true do
   describe '.stub' do
+    # Notice that this is referring to gRPC "stubs", not rspec stubs
     before { described_class.clear_stubs! }
 
     context 'when passed a UNIX socket address' do
@@ -32,4 +35,81 @@ describe Gitlab::GitalyClient, lib: true do
       end
     end
   end
+
+  describe 'feature_enabled?' do
+    let(:feature_name) { 'my_feature' }
+    let(:real_feature_name) { "gitaly_#{feature_name}" }
+
+    context 'when Gitaly is disabled' do
+      before { allow(described_class).to receive(:enabled?).and_return(false) }
+
+      it 'returns false' do
+        expect(described_class.feature_enabled?(feature_name)).to be(false)
+      end
+    end
+
+    context 'when the feature status is DISABLED' do
+      let(:feature_status) { Gitlab::GitalyClient::MigrationStatus::DISABLED }
+
+      it 'returns false' do
+        expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(false)
+      end
+    end
+
+    context 'when the feature_status is OPT_IN' do
+      let(:feature_status) { Gitlab::GitalyClient::MigrationStatus::OPT_IN }
+
+      context "when the feature flag hasn't been set" do
+        it 'returns false' do
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(false)
+        end
+      end
+
+      context "when the feature flag is set to disable" do
+        before { Feature.get(real_feature_name).disable }
+
+        it 'returns false' do
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(false)
+        end
+      end
+
+      context "when the feature flag is set to enable" do
+        before { Feature.get(real_feature_name).enable }
+
+        it 'returns true' do
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(true)
+        end
+      end
+
+      context "when the feature flag is set to a percentage of time" do
+        before { Feature.get(real_feature_name).enable_percentage_of_time(70) }
+
+        it 'bases the result on pseudo-random numbers' do
+          expect(Random).to receive(:rand).and_return(0.3)
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(true)
+
+          expect(Random).to receive(:rand).and_return(0.8)
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(false)
+        end
+      end
+    end
+
+    context 'when the feature_status is OPT_OUT' do
+      let(:feature_status) { Gitlab::GitalyClient::MigrationStatus::OPT_OUT }
+
+      context "when the feature flag hasn't been set" do
+        it 'returns true' do
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(true)
+        end
+      end
+
+      context "when the feature flag is set to disable" do
+        before { Feature.get(real_feature_name).disable }
+
+        it 'returns false' do
+          expect(described_class.feature_enabled?(feature_name, status: feature_status)).to be(false)
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -191,6 +191,7 @@ Ci::Pipeline:
 - lock_version
 - auto_canceled_by_id
 - pipeline_schedule_id
+- source
 CommitStatus:
 - id
 - project_id

spec/models/ci/pipeline_spec.rb

@@ -21,13 +21,35 @@ describe Ci::Pipeline, models: true do
   it { is_expected.to have_many(:auto_canceled_pipelines) }
   it { is_expected.to have_many(:auto_canceled_jobs) }
 
-  it { is_expected.to validate_presence_of :sha }
-  it { is_expected.to validate_presence_of :status }
+  it { is_expected.to validate_presence_of(:sha) }
+  it { is_expected.to validate_presence_of(:status) }
 
   it { is_expected.to respond_to :git_author_name }
   it { is_expected.to respond_to :git_author_email }
   it { is_expected.to respond_to :short_sha }
 
+  describe '#source' do
+    context 'when creating new pipeline' do
+      let(:pipeline) do
+        build(:ci_empty_pipeline, status: :created, project: project, source: nil)
+      end
+
+      it "prevents from creating an object" do
+        expect(pipeline).not_to be_valid
+      end
+    end
+
+    context 'when updating existing pipeline' do
+      before do
+        pipeline.update_attribute(:source, nil)
+      end
+
+      it "object is valid" do
+        expect(pipeline).to be_valid
+      end
+    end
+  end
+
   describe '#block' do
     it 'changes pipeline status to manual' do
       expect(pipeline.block).to be true

spec/requests/api/commit_statuses_spec.rb

@@ -16,8 +16,8 @@ describe API::CommitStatuses do
     let(:get_url) { "/projects/#{project.id}/repository/commits/#{sha}/statuses" }
 
     context 'ci commit exists' do
-      let!(:master) { project.pipelines.create(sha: commit.id, ref: 'master') }
-      let!(:develop) { project.pipelines.create(sha: commit.id, ref: 'develop') }
+      let!(:master) { project.pipelines.create(source: :push, sha: commit.id, ref: 'master') }
+      let!(:develop) { project.pipelines.create(source: :push, sha: commit.id, ref: 'develop') }
 
       context "reporter user" do
         let(:statuses_id) { json_response.map { |status| status['id'] } }

spec/requests/api/commits_spec.rb

@@ -485,7 +485,7 @@ describe API::Commits do
       end
 
       it "returns status for CI" do
-        pipeline = project.ensure_pipeline('master', project.repository.commit.sha)
+        pipeline = project.pipelines.create(source: :push, ref: 'master', sha: project.repository.commit.sha)
         pipeline.update(status: 'success')
 
         get api("/projects/#{project.id}/repository/commits/#{project.repository.commit.id}", user)
@@ -495,7 +495,7 @@ describe API::Commits do
       end
 
       it "returns status for CI when pipeline is created" do
-        project.ensure_pipeline('master', project.repository.commit.sha)
+        project.pipelines.create(source: :push, ref: 'master', sha: project.repository.commit.sha)
 
         get api("/projects/#{project.id}/repository/commits/#{project.repository.commit.id}", user)
 

spec/requests/api/features_spec.rb

+require 'spec_helper'
+
+describe API::Features do
+  let(:user)  { create(:user) }
+  let(:admin) { create(:admin) }
+
+  describe 'GET /features' do
+    let(:expected_features) do
+      [
+        {
+          'name' => 'feature_1',
+          'state' => 'on',
+          'gates' => [{ 'key' => 'boolean', 'value' => true }]
+        },
+        {
+          'name' => 'feature_2',
+          'state' => 'off',
+          'gates' => [{ 'key' => 'boolean', 'value' => false }]
+        }
+      ]
+    end
+
+    before do
+      Feature.get('feature_1').enable
+      Feature.get('feature_2').disable
+    end
+
+    it 'returns a 401 for anonymous users' do
+      get api('/features')
+
+      expect(response).to have_http_status(401)
+    end
+
+    it 'returns a 403 for users' do
+      get api('/features', user)
+
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns the feature list for admins' do
+      get api('/features', admin)
+
+      expect(response).to have_http_status(200)
+      expect(json_response).to match_array(expected_features)
+    end
+  end
+
+  describe 'POST /feature' do
+    let(:feature_name) { 'my_feature' }
+    it 'returns a 401 for anonymous users' do
+      post api("/features/#{feature_name}")
+
+      expect(response).to have_http_status(401)
+    end
+
+    it 'returns a 403 for users' do
+      post api("/features/#{feature_name}", user)
+
+      expect(response).to have_http_status(403)
+    end
+
+    it 'creates an enabled feature if passed true' do
+      post api("/features/#{feature_name}", admin), value: 'true'
+
+      expect(response).to have_http_status(201)
+      expect(Feature.get(feature_name)).to be_enabled
+    end
+
+    it 'creates a feature with the given percentage if passed an integer' do
+      post api("/features/#{feature_name}", admin), value: '50'
+
+      expect(response).to have_http_status(201)
+      expect(Feature.get(feature_name).percentage_of_time_value).to be(50)
+    end
+
+    context 'when the feature exists' do
+      let(:feature) { Feature.get(feature_name) }
+
+      before do
+        feature.disable # This also persists the feature on the DB
+      end
+
+      it 'enables the feature if passed true' do
+        post api("/features/#{feature_name}", admin), value: 'true'
+
+        expect(response).to have_http_status(201)
+        expect(feature).to be_enabled
+      end
+
+      context 'with a pre-existing percentage value' do
+        before do
+          feature.enable_percentage_of_time(50)
+        end
+
+        it 'updates the percentage of time if passed an integer' do
+          post api("/features/#{feature_name}", admin), value: '30'
+
+          expect(response).to have_http_status(201)
+          expect(Feature.get(feature_name).percentage_of_time_value).to be(30)
+        end
+      end
+    end
+  end
+end

spec/requests/api/v3/commits_spec.rb

@@ -386,7 +386,7 @@ describe API::V3::Commits do
       end
 
       it "returns status for CI" do
-        pipeline = project.ensure_pipeline('master', project.repository.commit.sha)
+        pipeline = project.pipelines.create(source: :push, ref: 'master', sha: project.repository.commit.sha)
         pipeline.update(status: 'success')
 
         get v3_api("/projects/#{project.id}/repository/commits/#{project.repository.commit.id}", user)
@@ -396,7 +396,7 @@ describe API::V3::Commits do
       end
 
       it "returns status for CI when pipeline is created" do
-        project.ensure_pipeline('master', project.repository.commit.sha)
+        project.pipelines.create(source: :push, ref: 'master', sha: project.repository.commit.sha)
 
         get v3_api("/projects/#{project.id}/repository/commits/#{project.repository.commit.id}", user)
 

spec/serializers/pipeline_details_entity_spec.rb

@@ -34,7 +34,7 @@ describe PipelineDetailsEntity do
       it 'contains flags' do
         expect(subject).to include :flags
         expect(subject[:flags])
-          .to include :latest, :triggered, :stuck,
+          .to include :latest, :stuck,
                       :yaml_errors, :retryable, :cancelable
       end
     end

spec/serializers/pipeline_entity_spec.rb

@@ -19,10 +19,24 @@ describe PipelineEntity do
       let(:pipeline) { create(:ci_empty_pipeline) }
 
       it 'contains required fields' do
-        expect(subject).to include :id, :user, :path, :coverage
+        expect(subject).to include :id, :user, :path, :coverage, :source
         expect(subject).to include :ref, :commit
         expect(subject).to include :updated_at, :created_at
       end
+
+      it 'contains details' do
+        expect(subject).to include :details
+        expect(subject[:details])
+          .to include :duration, :finished_at
+        expect(subject[:details][:status]).to include :icon, :favicon, :text, :label
+      end
+
+      it 'contains flags' do
+        expect(subject).to include :flags
+        expect(subject[:flags])
+          .to include :latest, :stuck,
+                      :yaml_errors, :retryable, :cancelable
+      end
     end
 
     context 'when pipeline is retryable' do

spec/services/ci/create_pipeline_service_spec.rb

@@ -9,13 +9,13 @@ describe Ci::CreatePipelineService, services: true do
   end
 
   describe '#execute' do
-    def execute_service(after: project.commit.id, message: 'Message', ref: 'refs/heads/master')
+    def execute_service(source: :push, after: project.commit.id, message: 'Message', ref: 'refs/heads/master')
       params = { ref: ref,
                  before: '00000000',
                  after: after,
                  commits: [{ message: message }] }
 
-      described_class.new(project, user, params).execute
+      described_class.new(project, user, params).execute(source)
     end
 
     context 'valid params' do
@@ -30,6 +30,7 @@ describe Ci::CreatePipelineService, services: true do
       it 'creates a pipeline' do
         expect(pipeline).to be_kind_of(Ci::Pipeline)
         expect(pipeline).to be_valid
+        expect(pipeline).to be_push
         expect(pipeline).to eq(project.pipelines.last)
         expect(pipeline).to have_attributes(user: user)
         expect(pipeline).to have_attributes(status: 'pending')

spec/services/ci/create_trigger_request_service_spec.rb

@@ -16,6 +16,7 @@ describe Ci::CreateTriggerRequestService, services: true do
       context 'without owner' do
         it { expect(subject).to be_kind_of(Ci::TriggerRequest) }
         it { expect(subject.pipeline).to be_kind_of(Ci::Pipeline) }
+        it { expect(subject.pipeline).to be_trigger }
         it { expect(subject.builds.first).to be_kind_of(Ci::Build) }
       end
 
@@ -25,6 +26,7 @@ describe Ci::CreateTriggerRequestService, services: true do
 
         it { expect(subject).to be_kind_of(Ci::TriggerRequest) }
         it { expect(subject.pipeline).to be_kind_of(Ci::Pipeline) }
+        it { expect(subject.pipeline).to be_trigger }
         it { expect(subject.pipeline.user).to eq(owner) }
         it { expect(subject.builds.first).to be_kind_of(Ci::Build) }
         it { expect(subject.builds.first.user).to eq(owner) }

spec/services/git_push_service_spec.rb

@@ -131,6 +131,19 @@ describe GitPushService, services: true do
     end
   end
 
+  describe "Pipelines" do
+    subject { execute_service(project, user, @oldrev, @newrev, @ref) }
+
+    before do
+      stub_ci_pipeline_to_return_yaml_file
+    end
+
+    it "creates a new pipeline" do
+      expect{ subject }.to change{ Ci::Pipeline.count }
+      expect(Ci::Pipeline.last).to be_push
+    end
+  end
+
   describe "Push Event" do
     before do
       service = execute_service(project, user, @oldrev, @newrev, @ref )

spec/services/git_tag_push_service_spec.rb

@@ -30,6 +30,20 @@ describe GitTagPushService, services: true do
     end
   end
 
+  describe "Pipelines" do
+    subject { service.execute }
+
+    before do
+      stub_ci_pipeline_to_return_yaml_file
+      project.team << [user, :developer]
+    end
+
+    it "creates a new pipeline" do
+      expect{ subject }.to change{ Ci::Pipeline.count }
+      expect(Ci::Pipeline.last).to be_push
+    end
+  end
+
   describe "Git Tag Push Data" do
     subject { @push_data }
     let(:tag) { project.repository.find_tag(tag_name) }

spec/support/gitaly.rb

 if Gitlab::GitalyClient.enabled?
   RSpec.configure do |config|
-    config.before(:each) do
+    config.before(:each) do |example|
+      next if example.metadata[:skip_gitaly_mock]
       allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(true)
     end
   end

spec/workers/pipeline_schedule_worker_spec.rb

@@ -23,7 +23,8 @@ describe PipelineScheduleWorker do
 
     context 'when there is a scheduled pipeline within next_run_at' do
       it 'creates a new pipeline' do
-        expect { subject }.to change { project.pipelines.count }.by(1)
+        expect{ subject }.to change { project.pipelines.count }.by(1)
+        expect(Ci::Pipeline.last).to be_schedule
       end
 
       it 'updates the next_run_at field' do