Merge branch 'ce-to-ee-2018-02-27' into 'master'

CE upstream - 2018-02-27 22:10 UTC

See merge request gitlab-org/gitlab-ee!4754

.codeclimate.yml

@@ -11,8 +11,8 @@ engines:
     exclude_paths:
       - "lib/api/v3/*"
   eslint:
-    # eslint-plugin-vue is locked to version 2 in codeclimate, we need version 4
-    enabled: false
+    enabled: true
+    channel: "eslint-4"
   rubocop:
     enabled: true
     channel: "gitlab-rubocop-0-52-1"

app/assets/javascripts/labels_select.js

@@ -242,10 +242,16 @@ export default class LabelsSelect {
         filterable: true,
         selected: $dropdown.data('selected') || [],
         toggleLabel: function(selected, el) {
+          var $dropdownParent = $dropdown.parent();
+          var $dropdownInputField = $dropdownParent.find('.dropdown-input-field');
           var isSelected = el !== null ? el.hasClass('is-active') : false;
           var title = selected.title;
           var selectedLabels = this.selected;
 
+          if ($dropdownInputField.length && $dropdownInputField.val().length) {
+            $dropdownParent.find('.dropdown-input-clear').trigger('click');
+          }
+
           if (selected.id === 0) {
             this.selected = [];
             return 'No Label';

app/assets/javascripts/pages/projects/issues/show.js

+import initIssuableSidebar from '~/init_issuable_sidebar';
+import Issue from '~/issue';
+import ShortcutsIssuable from '~/shortcuts_issuable';
+import ZenMode from '~/zen_mode';
+import '~/notes/index';
+import '~/issue_show/index';
+
+export default function () {
+  new Issue(); // eslint-disable-line no-new
+  new ShortcutsIssuable(); // eslint-disable-line no-new
+  new ZenMode(); // eslint-disable-line no-new
+  initIssuableSidebar();
+}

app/assets/javascripts/pages/projects/issues/show/index.js

-import initIssuableSidebar from '~/init_issuable_sidebar';
-import Issue from '~/issue';
-import ShortcutsIssuable from '~/shortcuts_issuable';
-import ZenMode from '~/zen_mode';
-import '~/notes/index';
-import '~/issue_show/index';
+import initSidebarBundle from '~/sidebar/sidebar_bundle';
+import initShow from '../show';
 
 document.addEventListener('DOMContentLoaded', () => {
-  new Issue(); // eslint-disable-line no-new
-  new ShortcutsIssuable(); // eslint-disable-line no-new
-  new ZenMode(); // eslint-disable-line no-new
-  initIssuableSidebar();
+  initShow();
+  initSidebarBundle();
 });

app/assets/javascripts/pages/projects/merge_requests/conflicts/index.js

+import initSidebarBundle from '~/sidebar/sidebar_bundle';
+
+document.addEventListener('DOMContentLoaded', initSidebarBundle);

app/assets/javascripts/pages/projects/merge_requests/conflicts/show/index.js

+import initSidebarBundle from '~/sidebar/sidebar_bundle';
+
+document.addEventListener('DOMContentLoaded', initSidebarBundle);

app/assets/javascripts/pages/projects/merge_requests/init_merge_request_show.js

+import MergeRequest from '~/merge_request';
+import ZenMode from '~/zen_mode';
+import initNotes from '~/init_notes';
+import initIssuableSidebar from '~/init_issuable_sidebar';
+import initDiffNotes from '~/diff_notes/diff_notes_bundle';
+import ShortcutsIssuable from '~/shortcuts_issuable';
+import Diff from '~/diff';
+import { handleLocationHash } from '~/lib/utils/common_utils';
+import howToMerge from '~/how_to_merge';
+import initPipelines from '~/commit/pipelines/pipelines_bundle';
+import initWidget from '../../../vue_merge_request_widget';
+
+export default function () {
+  new Diff(); // eslint-disable-line no-new
+  new ZenMode(); // eslint-disable-line no-new
+
+  initIssuableSidebar();
+  initNotes();
+  initDiffNotes();
+  initPipelines();
+
+  const mrShowNode = document.querySelector('.merge-request');
+
+  window.mergeRequest = new MergeRequest({
+    action: mrShowNode.dataset.mrAction,
+  });
+
+  new ShortcutsIssuable(true); // eslint-disable-line no-new
+  handleLocationHash();
+  howToMerge();
+  initWidget();
+}

app/assets/javascripts/pages/projects/merge_requests/show/index.js

-import MergeRequest from '~/merge_request';
-import ZenMode from '~/zen_mode';
-import initNotes from '~/init_notes';
-import initIssuableSidebar from '~/init_issuable_sidebar';
-import initDiffNotes from '~/diff_notes/diff_notes_bundle';
-import ShortcutsIssuable from '~/shortcuts_issuable';
-import Diff from '~/diff';
-import { handleLocationHash } from '~/lib/utils/common_utils';
-import howToMerge from '~/how_to_merge';
-import initPipelines from '~/commit/pipelines/pipelines_bundle';
-import initWidget from '../../../../vue_merge_request_widget';
+import initSidebarBundle from '~/sidebar/sidebar_bundle';
+import initShow from '../init_merge_request_show';
 
 document.addEventListener('DOMContentLoaded', () => {
-  new Diff(); // eslint-disable-line no-new
-  new ZenMode(); // eslint-disable-line no-new
-
-  initIssuableSidebar();
-  initNotes();
-  initDiffNotes();
-  initPipelines();
-
-  const mrShowNode = document.querySelector('.merge-request');
-  window.mergeRequest = new MergeRequest({
-    action: mrShowNode.dataset.mrAction,
-  });
-
-  new ShortcutsIssuable(true); // eslint-disable-line no-new
-  handleLocationHash();
-  howToMerge();
-  initWidget();
+  initShow();
+  initSidebarBundle();
 });

app/assets/javascripts/sidebar/sidebar_bundle.js

 import Mediator from './sidebar_mediator';
 import { mountSidebar, getSidebarOptions } from './mount_sidebar';
 
-function domContentLoaded() {
+export default () => {
   const mediator = new Mediator(getSidebarOptions());
   mediator.fetch();
 
   mountSidebar(mediator);
-}
-
-document.addEventListener('DOMContentLoaded', domContentLoaded);
-
-export default domContentLoaded;
+};

app/models/user.rb

@@ -449,7 +449,7 @@ class User < ActiveRecord::Base
   end
 
   def self.non_internal
-    where(Hash[internal_attributes.zip([[false, nil]] * internal_attributes.size)])
+    where(internal_attributes.map { |attr| "#{attr} IS NOT TRUE" }.join(" AND "))
   end
 
   #

app/views/projects/project_members/update.js.haml

-:plain
-  var $listItem = $('#{escape_javascript(render('shared/members/member', member: @project_member))}');
-  $("##{dom_id(@project_member)} .list-item-name").replaceWith($listItem.find('.list-item-name'));
-  gl.utils.localTimeAgo($('.js-timeago'), $("##{dom_id(@project_member)}"));

changelogs/unreleased/24774-clear-the-Labels-dropdown-search-filter.yml

+---
+title: Clear the Labels dropdown search filter after a selection is made
+merge_request: 17393
+author: Andrew Torres
+type: changed

changelogs/unreleased/40502-osw-keep-link-when-redacting-unauthorized-objects.yml

+---
+title: Keep link when redacting unauthorized object links
+merge_request:
+author:
+type: fixed

changelogs/unreleased/41851-enable-eslint-codeclimate.yml

+---
+title: Enables eslint in codeclimate job
+merge_request: 17392
+author:
+type: other

changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml

+---
+title: Add catch-up background migration to migrate pipeline stages
+merge_request: 15741
+author:
+type: performance

config/webpack.config.js

@@ -385,7 +385,7 @@ if (IS_DEV_SERVER) {
           callback();
         })
       },
-    },
+    }
   );
   if (DEV_SERVER_LIVERELOAD) {
     config.plugins.push(new webpack.HotModuleReplacementPlugin());

db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb

+class AddTmpPartialNullIndexToBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+                                          name: 'tmp_id_partial_null_index')
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+  end
+end

db/post_migrate/20180212101928_schedule_build_stage_migration.rb

+class ScheduleBuildStageMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateBuildStage'.freeze
+  BATCH_SIZE = 500
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+  end
+
+  def up
+    disable_statement_timeout
+
+    Build.where('stage_id IS NULL').tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # noop
+  end
+end

db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb

+class RemoveTmpPartialNullIndexFromBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+  end
+
+  def down
+    add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+                                          name: 'tmp_id_partial_null_index')
+  end
+end

ee/app/assets/javascripts/pages/projects/issues/show/index.js

-import '~/pages/projects/issues/show/index';
+import initShow from '~/pages/projects/issues/show';
 import initSidebarBundle from 'ee/sidebar/sidebar_bundle';
 
-document.addEventListener('DOMContentLoaded', initSidebarBundle);
+document.addEventListener('DOMContentLoaded', () => {
+  initShow();
+  initSidebarBundle();
+});

ee/app/assets/javascripts/pages/projects/merge_requests/show/index.js

-import '~/pages/projects/merge_requests/show/index';
+import initShow from '~/pages/projects/merge_requests/init_merge_request_show';
 import initSidebarBundle from 'ee/sidebar/sidebar_bundle';
 
-document.addEventListener('DOMContentLoaded', initSidebarBundle);
+document.addEventListener('DOMContentLoaded', () => {
+  initShow();
+  initSidebarBundle();
+});

ee/lib/ee/banzai/filter/epic_reference_filter.rb

@@ -26,10 +26,11 @@ module EE
           urls.group_epic_url(group, epic, only_path: context[:only_path])
         end
 
-        def data_attributes_for(text, group, object, link: false)
+        def data_attributes_for(text, group, object, link_content: false, link_reference: false)
           data_attribute(
             original:       text,
-            link:         link,
+            link:           link_content,
+            link_reference: link_reference,
             group:          group.id,
             object_sym =>   object.id
           )

lib/banzai/filter/abstract_reference_filter.rb

@@ -174,7 +174,9 @@ module Banzai
             title = object_link_title(object)
             klass = reference_class(object_sym)
 
-            data = data_attributes_for(link_content || match, parent, object, link: !!link_content)
+            data = data_attributes_for(link_content || match, parent, object,
+                                       link_content: !!link_content,
+                                       link_reference: link_reference)
 
             url =
               if matches.names.include?("url") && matches[:url]
@@ -194,10 +196,11 @@ module Banzai
         end
       end
 
-      def data_attributes_for(text, project, object, link: false)
+      def data_attributes_for(text, project, object, link_content: false, link_reference: false)
         data_attribute(
           original:       text,
-          link:         link,
+          link:           link_content,
+          link_reference: link_reference,
           project:        project.id,
           object_sym =>   object.id
         )

lib/banzai/redactor.rb

@@ -42,16 +42,33 @@ module Banzai
           next if visible.include?(node)
 
           doc_data[:visible_reference_count] -= 1
-          # The reference should be replaced by the original link's content,
-          # which is not always the same as the rendered one.
-          content = node.attr('data-original') || node.inner_html
-          node.replace(content)
+          redacted_content = redacted_node_content(node)
+          node.replace(redacted_content)
         end
       end
 
       metadata
     end
 
+    # Return redacted content of given node as either the original link (<a> tag),
+    # the original content (text), or the inner HTML of the node.
+    #
+    def redacted_node_content(node)
+      original_content = node.attr('data-original')
+      link_reference = node.attr('data-link-reference')
+
+      # Build the raw <a> tag just with a link as href and content if
+      # it's originally a link pattern. We shouldn't return a plain text href.
+      original_link =
+        if link_reference == 'true' && href = original_content
+          %(<a href="#{href}">#{href}</a>)
+        end
+
+      # The reference should be replaced by the original link's content,
+      # which is not always the same as the rendered one.
+      original_link || original_content || node.inner_html
+    end
+
     def redact_cross_project_references(documents)
       extractor = Banzai::IssuableExtractor.new(project, user)
       issuables = extractor.extract(documents)

lib/gitlab/background_migration/migrate_build_stage.rb

+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateBuildStage
+      module Migratable
+        class Stage < ActiveRecord::Base
+          self.table_name = 'ci_stages'
+        end
+
+        class Build < ActiveRecord::Base
+          self.table_name = 'ci_builds'
+
+          def ensure_stage!(attempts: 2)
+            find_stage || create_stage!
+          rescue ActiveRecord::RecordNotUnique
+            retry if (attempts -= 1) > 0
+            raise
+          end
+
+          def find_stage
+            Stage.find_by(name: self.stage || 'test',
+                          pipeline_id: self.commit_id,
+                          project_id: self.project_id)
+          end
+
+          def create_stage!
+            Stage.create!(name: self.stage || 'test',
+                          pipeline_id: self.commit_id,
+                          project_id: self.project_id)
+          end
+        end
+      end
+
+      def perform(start_id, stop_id)
+        stages = Migratable::Build.where('stage_id IS NULL')
+          .where('id BETWEEN ? AND ?', start_id, stop_id)
+          .map { |build| build.ensure_stage! }
+          .compact.map(&:id)
+
+        MigrateBuildStageIdReference.new.perform(start_id, stop_id)
+        MigrateStageStatus.new.perform(stages.min, stages.max)
+      end
+    end
+  end
+end

spec/features/issues/form_spec.rb

@@ -193,6 +193,18 @@ describe 'New/edit issue', :js do
       expect(find('.js-label-select')).to have_content('Labels')
     end
 
+    it 'clears label search input field when a label is selected' do
+      click_button 'Labels'
+
+      page.within '.dropdown-menu-labels' do
+        search_field = find('input[type="search"]')
+
+        search_field.set(label2.title)
+        click_link label2.title
+        expect(search_field.value).to eq ''
+      end
+    end
+
     it 'correctly updates the selected user when changing assignee' do
       click_button 'Unassigned'
 

spec/lib/banzai/redactor_spec.rb

@@ -40,6 +40,16 @@ describe Banzai::Redactor do
           expect(doc.to_html).to eq(original_content)
         end
       end
+
+      it 'returns <a> tag with original href if it is originally a link reference' do
+        href = 'http://localhost:3000'
+        doc = Nokogiri::HTML
+          .fragment("<a class='gfm' data-reference-type='issue' data-original=#{href} data-link-reference='true'>#{href}</a>")
+
+        redactor.redact([doc])
+
+        expect(doc.to_html).to eq('<a href="http://localhost:3000">http://localhost:3000</a>')
+      end
     end
 
     context 'when project is in pending delete' do

spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateBuildStage, :migration, schema: 20180212101928 do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+               failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+  before do
+    projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+
+    jobs.create!(id: 1, commit_id: 1, project_id: 123,
+                 stage_idx: 2, stage: 'build', status: :success)
+    jobs.create!(id: 2, commit_id: 1, project_id: 123,
+                 stage_idx: 2, stage: 'build', status: :success)
+    jobs.create!(id: 3, commit_id: 1, project_id: 123,
+                 stage_idx: 1, stage: 'test', status: :failed)
+    jobs.create!(id: 4, commit_id: 1, project_id: 123,
+                 stage_idx: 1, stage: 'test', status: :success)
+    jobs.create!(id: 5, commit_id: 1, project_id: 123,
+                 stage_idx: 3, stage: 'deploy', status: :pending)
+    jobs.create!(id: 6, commit_id: 1, project_id: 123,
+                 stage_idx: 3, stage: nil, status: :pending)
+  end
+
+  it 'correctly migrates builds stages' do
+    expect(stages.count).to be_zero
+
+    described_class.new.perform(1, 6)
+
+    expect(stages.count).to eq 3
+    expect(stages.all.pluck(:name)).to match_array %w[test build deploy]
+    expect(jobs.where(stage_id: nil)).to be_one
+    expect(jobs.find_by(stage_id: nil).id).to eq 6
+    expect(stages.all.pluck(:status)).to match_array [STATUSES[:success],
+                                                      STATUSES[:failed],
+                                                      STATUSES[:pending]]
+  end
+
+  it 'recovers from unique constraint violation only twice' do
+    allow(described_class::Migratable::Stage)
+      .to receive(:find_by).and_return(nil)
+
+    expect(described_class::Migratable::Stage)
+      .to receive(:find_by).exactly(3).times
+
+    expect { described_class.new.perform(1, 6) }
+      .to raise_error ActiveRecord::RecordNotUnique
+  end
+end

spec/migrations/schedule_build_stage_migration_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180212101928_schedule_build_stage_migration')
+
+describe ScheduleBuildStageMigration, :migration do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+
+    projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    stages.create!(id: 1, project_id: 123, pipeline_id: 1, name: 'test')
+
+    jobs.create!(id: 11, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 206, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 3413, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 4109, commit_id: 1, project_id: 123, stage_id: 1)
+  end
+
+  it 'schedules delayed background migrations in batches in bulk' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 11, 11)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 206, 206)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 3413, 3413)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end