Merge branch 'master' into rename-reply-by-email

CHANGELOG

 Please view this file on the master branch, on stable branches it's out of date.
 
 v 8.0.0 (unreleased)
+  - Fix Markdown links not showing up in dashboard activity feed (Stan Hu)
   - Fix HTML link that was improperly escaped in new user e-mail (Stan Hu)
   - Fix broken sort in merge request API (Stan Hu)
   - Bump rouge to 1.10.1 to remove warning noise and fix other syntax highlighting bugs (Stan Hu)
@@ -42,6 +43,7 @@ v 8.0.0 (unreleased)
   - Retrieving oauth token with LDAP credentials
   - Load Application settings from running database unless env var USE_DB=false
   - Added Drone CI integration (Kirill Zaitsev)
+  - Fail builds if no .gitlab-ci.yml is found
   - Refactored service API and added automatically service docs generator (Kirill Zaitsev)
   - Added web_url key project hook_attrs (Kirill Zaitsev)
   - Add ability to get user information by ID of an SSH key via the API

app/helpers/gitlab_markdown_helper.rb

@@ -45,7 +45,7 @@ module GitlabMarkdownHelper
   end
 
   def markdown(text, context = {})
-    context.merge!(
+    context.reverse_merge!(
       current_user: current_user,
       path:         @path,
       project:      @project,
@@ -59,7 +59,7 @@ module GitlabMarkdownHelper
   # TODO (rspeicher): Remove all usages of this helper and just call `markdown`
   # with a custom pipeline depending on the content being rendered
   def gfm(text, options = {})
-    options.merge!(
+    options.reverse_merge!(
       current_user: current_user,
       path:         @path,
       project:      @project,

app/mailers/emails/notes.rb

@@ -12,7 +12,7 @@ module Emails
                          to: recipient(recipient_id),
                          subject: subject("#{@commit.title} (#{@commit.short_id})"))
 
-      SentNotification.record(@commit, recipient_id, reply_key)
+      SentNotification.record_note(@note, recipient_id, reply_key)
     end
 
     def note_issue_email(recipient_id, note_id)
@@ -27,7 +27,7 @@ module Emails
                          to: recipient(recipient_id),
                          subject: subject("#{@issue.title} (##{@issue.iid})"))
 
-      SentNotification.record(@issue, recipient_id, reply_key)
+      SentNotification.record_note(@note, recipient_id, reply_key)
     end
 
     def note_merge_request_email(recipient_id, note_id)
@@ -43,7 +43,7 @@ module Emails
                          to: recipient(recipient_id),
                          subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
 
-      SentNotification.record(@merge_request, recipient_id, reply_key)
+      SentNotification.record_note(@note, recipient_id, reply_key)
     end
   end
 end

app/models/ci/commit.rb

@@ -236,7 +236,7 @@ module Ci
     end
 
     def config_processor
-      @config_processor ||= Ci::GitlabCiYamlProcessor.new(push_data[:ci_yaml_file] || project.generated_yaml_config)
+      @config_processor ||= Ci::GitlabCiYamlProcessor.new(push_data[:ci_yaml_file])
     rescue Ci::GitlabCiYamlProcessor::ValidationError => e
       save_yaml_error(e.message)
       nil

app/models/project.rb

@@ -428,7 +428,7 @@ class Project < ActiveRecord::Base
   end
 
   def gitlab_ci?
-    gitlab_ci_service && gitlab_ci_service.active
+    gitlab_ci_service && gitlab_ci_service.active && gitlab_ci_project.present?
   end
 
   def ci_services

app/models/sent_notification.rb

@@ -8,6 +8,7 @@
 #  noteable_type :string(255)
 #  recipient_id  :integer
 #  commit_id     :string(255)
+#  line_code     :string(255)
 #  reply_key     :string(255)      not null
 #
 
@@ -21,6 +22,7 @@ class SentNotification < ActiveRecord::Base
 
   validates :noteable_id, presence: true, unless: :for_commit?
   validates :commit_id, presence: true, if: :for_commit?
+  validates :line_code, format: { with: /\A[a-z0-9]+_\d+_\d+\Z/ }, allow_blank: true
 
   class << self
     def reply_key
@@ -33,7 +35,7 @@ class SentNotification < ActiveRecord::Base
       find_by(reply_key: reply_key)
     end
 
-    def record(noteable, recipient_id, reply_key)
+    def record(noteable, recipient_id, reply_key, params = {})
       return unless reply_key
 
       noteable_id = nil
@@ -44,7 +46,7 @@ class SentNotification < ActiveRecord::Base
         noteable_id = noteable.id
       end
 
-      create(
+      params.reverse_merge!(
         project:        noteable.project,
         noteable_type:  noteable.class.name,
         noteable_id:    noteable_id,
@@ -52,6 +54,14 @@ class SentNotification < ActiveRecord::Base
         recipient_id:   recipient_id,
         reply_key:      reply_key
       )
+
+      create(params)
+    end
+
+    def record_note(note, recipient_id, reply_key, params = {})
+      params[:line_code] = note.line_code
+      
+      record(note.noteable, recipient_id, reply_key, params)
     end
   end
 

app/views/admin/application_settings/_form.html.haml

@@ -131,7 +131,7 @@
         .checkbox
           = f.label :ci_enabled do
             = f.check_box :ci_enabled
-            Enable Continuous Integration
+            Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
 
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'

app/views/ci/projects/disabled.html.haml

-Continuous Integration has been disabled. Please ask your administrator to enable it.
+Continuous Integration has been disabled for time of the migration.

db/migrate/20150920161119_add_line_code_to_sent_notification.rb

+class AddLineCodeToSentNotification < ActiveRecord::Migration
+  def change
+    add_column :sent_notifications, :line_code, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150920010715) do
+ActiveRecord::Schema.define(version: 20150920161119) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -623,6 +623,7 @@ ActiveRecord::Schema.define(version: 20150920010715) do
     t.integer "recipient_id"
     t.string  "commit_id"
     t.string  "reply_key",     null: false
+    t.string  "line_code"
   end
 
   add_index "sent_notifications", ["reply_key"], name: "index_sent_notifications_on_reply_key", unique: true, using: :btree

doc/integration/README.md

@@ -10,7 +10,7 @@ See the documentation below for details on how to configure these services.
 - [SAML](saml.md) Configure GitLab as a SAML 2.0 Service Provider
 - [Slack](slack.md) Integrate with the Slack chat service
 - [OAuth2 provider](oauth_provider.md) OAuth2 application creation
-- [Gmail](gitlab_buttons_in_gmail.md) Adds GitLab actions to messages
+- [Gmail actions buttons](gmail_action_buttons_for_gitlab.md) Adds GitLab actions to messages
 
 GitLab Enterprise Edition contains [advanced JIRA support](http://doc.gitlab.com/ee/integration/jira.html) and [advanced Jenkins support](http://doc.gitlab.com/ee/integration/jenkins.html).
 

doc/integration/gmail_action_buttons_for_gitlab.md

+# Gmail actions buttons for GitLab
+
+GitLab supports [Google actions in email](https://developers.google.com/gmail/markup/actions/actions-overview).
+
+If correctly setup, emails that require an action will be marked in Gmail.
+
+![gmail_actions_button.png](gmail_actions_button.png)
+
+To get this functioning, you need to be registered with Google.
+[See how to register with Google in this document.](https://developers.google.com/gmail/markup/registering-with-google)
+
+*This process has a lot of steps so make sure that you fulfill all requirements set by Google.*
+*Your application will be rejected by Google if you fail to do so.*
+
+Pay close attention to:
+
+* Email account used by GitLab to send notification emails needs to have "Consistent history of sending a high volume of mail from your domain (order of hundred emails a day minimum to Gmail) for a few weeks at least".
+* "A very very low rate of spam complaints from users."
+* Emails must be authenticated via DKIM or SPF
+* Before sending the final form("Gmail Schema Whitelist Request"), you must send a real email from your production server. This means that you will have to find a way to send this email from the email address you are registering. You can do this by, for example, forwarding the real email from the email address you are registering or going into the rails console on the GitLab server and triggering the email sending from there.
+
+You can check how it looks going through all the steps laid out in the "Registering with Google" doc in [this GitLab.com issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/1517).

lib/ci/migrate/builds.rb

+module Ci
+  module Migrate
+    class Builds
+      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+      def initialize
+        @app_builds_dir = Settings.gitlab_ci.builds_path
+        @backup_dir = Gitlab.config.backup.path
+        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+      end
+
+      def restore
+        backup_existing_builds_dir
+
+        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+        unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
+          abort 'Restore failed'.red
+        end
+      end
+
+      def backup_existing_builds_dir
+        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+        if File.exists?(app_builds_dir)
+          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+        end
+      end
+    end
+  end
+end

lib/ci/migrate/database.rb

@@ -9,32 +9,32 @@ module Ci
         @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       end
 
-      def restore(ci_dump)
-        puts 'Deleting all CI related data ... '
-        truncate_ci_tables
+      def restore
+        decompress_rd, decompress_wr = IO.pipe
+        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+        decompress_wr.close
 
-        puts 'Restoring CI data ... '
-        case config["adapter"]
+        restore_pid = case config["adapter"]
                         when /^mysql/ then
-            print "Restoring MySQL database #{config['database']} ... "
+                          $progress.print "Restoring MySQL database #{config['database']} ... "
                           # Workaround warnings from MySQL 5.6 about passwords on cmd line
                           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-            system('mysql', *mysql_args, config['database'], in: ci_dump)
+                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
                         when "postgresql" then
-            puts "Restoring PostgreSQL database #{config['database']} ... "
+                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
                           pg_env
-            system('psql', config['database'], '-f', ci_dump)
+                          spawn('psql', config['database'], in: decompress_rd)
                       end
+        decompress_rd.close
+
+        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+        abort 'Restore failed' unless success
       end
 
       protected
 
-      def truncate_ci_tables
-        c = ActiveRecord::Base.connection
-        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-          puts "Deleting data from #{table}..."
-          c.execute("DELETE FROM #{table}")
-        end
+      def db_file_name
+        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
       end
 
       def mysql_args

lib/ci/migrate/manager.rb

+module Ci
+  module Migrate
+    class Manager
+      VERSION = '8.0.0.pre'
+
+      def cleanup
+        $progress.print "Deleting tmp directories ... "
+
+        backup_contents.each do |dir|
+          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+            $progress.puts "done".green
+          else
+            puts "deleting tmp directory '#{dir}' failed".red
+            abort 'Backup failed'
+          end
+        end
+      end
+
+      def unpack
+        Dir.chdir(Gitlab.config.backup.path)
+
+        # check for existing backups in the backup dir
+        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+        puts "no backups found" if file_list.count == 0
+
+        if file_list.count > 1 && ENV["BACKUP"].nil?
+          puts "Found more than one backup, please specify which one you want to restore:"
+          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+          exit 1
+        end
+
+        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+        unless File.exists?(tar_file)
+          puts "The specified CI backup doesn't exist!"
+          exit 1
+        end
+
+        $progress.print "Unpacking backup ... "
+
+        unless Kernel.system(*%W(tar -xf #{tar_file}))
+          puts "unpacking backup failed".red
+          exit 1
+        else
+          $progress.puts "done".green
+        end
+
+        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != VERSION
+          puts "GitLab CI version mismatch:".red
+          puts "  Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+          exit 1
+        end
+      end
+
+      private
+
+      def backup_contents
+        ["db", "builds", "backup_information.yml"]
+      end
+
+      def settings
+        @settings ||= YAML.load_file("backup_information.yml")
+      end
+    end
+  end
+end
+

lib/ci/migrate/tags.rb

@@ -4,45 +4,38 @@ module Ci
   module Migrate
     class Tags
       def restore
-        puts 'Migrating tags for Runners... '
-        list_objects('Runner').each do |id|
-          putc '.'
-          runner = Ci::Runner.find_by_id(id)
-          if runner
-            tags = list_tags('Runner', id)
-            runner.update_attributes(tag_list: tags)
+        puts 'Inserting tags...'
+        connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
+          begin
+            connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
+          rescue ActiveRecord::RecordNotUnique
           end
         end
-        puts ''
 
-        puts 'Migrating tags for Builds... '
-        list_objects('Build').each do |id|
-          putc '.'
-          build = Ci::Build.find_by_id(id)
-          if build
-            tags = list_tags('Build', id)
-            build.update_attributes(tag_list: tags)
-          end
-        end
-        puts ''
-      end
+        ActiveRecord::Base.transaction do
+          puts 'Deleting old taggings...'
+          connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
 
-      protected
+          puts 'Inserting taggings...'
+          connection.execute(
+            'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
+              "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
+              'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
+              'JOIN tags ON tags.name = ci_tags.name '
+          )
 
-      def list_objects(type)
-        ids = ActiveRecord::Base.connection.select_all(
-          "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
+          puts 'Resetting counters... '
+          connection.execute(
+            'UPDATE tags SET ' +
+              'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
           )
-        ids.map { |id| id['taggable_id'] }
+        end
       end
 
-      def list_tags(type, id)
-        tags = ActiveRecord::Base.connection.select_all(
-          'select ci_tags.name from ci_tags ' +
-            'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
-            "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
-        )
-        tags.map { |tag| tag['name'] }
+      protected
+
+      def connection
+        ActiveRecord::Base.connection
       end
     end
   end

lib/tasks/ci/migrate.rake

 namespace :ci do
   desc 'GitLab | Import and migrate CI database'
   task migrate: :environment do
+    warn_user_is_not_gitlab
+    configure_cron_mode
+
     unless ENV['force'] == 'yes'
-      puts "This will truncate all CI tables and restore it from provided backup."
-      puts "You will lose any previous CI data stored in the database."
+      puts 'This will remove all CI related data and restore it from the provided backup.'
       ask_to_continue
-      puts ""
+      puts ''
     end
 
-    Rake::Task["ci:migrate:db"].invoke
-    Rake::Task["ci:migrate:autoincrements"].invoke
-    Rake::Task["ci:migrate:tags"].invoke
-    Rake::Task["ci:migrate:services"].invoke
+    # disable CI for time of migration
+    enable_ci(false)
+
+    # unpack archives
+    migrate = Ci::Migrate::Manager.new
+    migrate.unpack
+
+    Rake::Task['ci:migrate:db'].invoke
+    Rake::Task['ci:migrate:builds'].invoke
+    Rake::Task['ci:migrate:tags'].invoke
+    Rake::Task['ci:migrate:services'].invoke
+
+    # enable CI for time of migration
+    enable_ci(true)
+
+    migrate.cleanup
   end
 
   namespace :migrate do
     desc 'GitLab | Import CI database'
     task db: :environment do
-      if ENV["CI_DUMP"].nil?
-        puts "No CI SQL dump specified:"
-        puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
-        exit 1
+      configure_cron_mode
+      $progress.puts 'Restoring database ... '.blue
+      Ci::Migrate::Database.new.restore
+      $progress.puts 'done'.green
     end
 
-      ci_dump = ENV["CI_DUMP"]
-      unless File.exists?(ci_dump)
-        puts "The specified sql dump doesn't exist!"
-        exit 1
-      end
-
-      ::Ci::Migrate::Database.new.restore(ci_dump)
+    desc 'GitLab | Import CI builds'
+    task builds: :environment do
+      configure_cron_mode
+      $progress.puts 'Restoring builds ... '.blue
+      Ci::Migrate::Builds.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI tags'
     task tags: :environment do
+      configure_cron_mode
+      $progress.puts 'Migrating tags ... '.blue
       ::Ci::Migrate::Tags.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI auto-increments'
@@ -56,8 +72,16 @@ namespace :ci do
 
     desc 'GitLab | Migrate CI services'
     task services: :environment do
+      $progress.puts 'Migrating services ... '.blue
       c = ActiveRecord::Base.connection
       c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
+      $progress.puts 'done'.green
     end
   end
+
+  def enable_ci(enabled)
+    settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
+    settings.ci_enabled = enabled
+    settings.save!
+  end
 end

spec/helpers/gitlab_markdown_helper_spec.rb

@@ -38,6 +38,17 @@ describe GitlabMarkdownHelper do
         expect(markdown(actual)).to match(expected)
       end
     end
+
+    describe "override default project" do
+      let(:actual) { issue.to_reference }
+      let(:second_project) { create(:project) }
+      let(:second_issue) { create(:issue, project: second_project) }
+
+      it 'should link to the issue' do
+        expected = namespace_project_issue_path(second_project.namespace, second_project, second_issue)
+        expect(markdown(actual, project: second_project)).to match(expected)
+      end
+    end
   end
 
   describe '#link_to_gfm' do

spec/services/ci/create_commit_service_spec.rb

@@ -50,6 +50,19 @@ module Ci
         end
       end
 
+      it 'fails commits without .gitlab-ci.yml' do
+        result = service.execute(project,
+                                 ref: 'refs/heads/0_1',
+                                 before: '00000000',
+                                 after: '31das312',
+                                 ci_yaml_file: config,
+                                 commits: [ { message: 'Message' } ]
+        )
+        expect(result).to be_persisted
+        expect(result.builds.any?).to be_falsey
+        expect(result.status).to eq('failed')
+      end
+
       describe :ci_skip? do
         it "skips builds creation if there is [ci skip] tag in commit message" do
           commits = [{ message: "some message[ci skip]" }]