Add latest changes from gitlab-org/gitlab@master

.gitignore

@@ -30,6 +30,7 @@ eslint-report.html
 /app/assets/javascripts/locale/**/app.js
 /backups/*
 /config/aws.yml
+/config/cable.yml
 /config/database*.yml
 /config/gitlab.yml
 /config/gitlab_ci.yml

app/assets/javascripts/pages/admin/index.js

 import initAdmin from './admin';
 import initAdminStatisticsPanel from '../../admin/statistics_panel/index';
+import initVueAlerts from '../../vue_alerts';
+
+document.addEventListener('DOMContentLoaded', initVueAlerts);
 
 document.addEventListener('DOMContentLoaded', () => {
   const statisticsPanelContainer = document.getElementById('js-admin-statistics-container');

app/controllers/admin/dashboard_controller.rb

@@ -12,6 +12,7 @@ class Admin::DashboardController < Admin::ApplicationController
     @projects = Project.order_id_desc.without_deleted.with_route.limit(10)
     @users = User.order_id_desc.limit(10)
     @groups = Group.order_id_desc.with_route.limit(10)
+    @notices = Gitlab::ConfigChecker::PumaRuggedChecker.check
   end
   # rubocop: enable CodeReuse/ActiveRecord
 

app/views/admin/dashboard/index.html.haml

@@ -3,6 +3,12 @@
 - if show_license_breakdown?
   = render_if_exists 'admin/licenses/breakdown', license: @license
 
+- if @notices
+  - @notices.each do |notice|
+    .js-vue-alert{ 'v-cloak': true, data: { variant: notice[:type],
+      dismissible: true.to_s } }
+      = notice[:message].html_safe
+
 .admin-dashboard.prepend-top-default
   .row
     .col-sm-4

changelogs/unreleased/create-puma-rugged-suboptimal-config-checker.yml

+---
+title: 'Show notices in Admin area when detected any of these cases: Puma, multi-threaded
+  Puma, multi-threaded Puma + Rugged'
+merge_request: 21403
+author:
+type: added

lib/banzai/filter/inline_embeds_filter.rb

@@ -6,6 +6,7 @@ module Banzai
     # a given link format. To transform references to DB
     # resources in place, prefer to inherit from AbstractReferenceFilter.
     class InlineEmbedsFilter < HTML::Pipeline::Filter
+      include Gitlab::Utils::StrongMemoize
       # Find every relevant link, create a new node based on
       # the link, and insert this node after any html content
       # surrounding the link.
@@ -60,6 +61,16 @@ module Banzai
 
         link_pattern.match(url) { |m| m.named_captures }
       end
+
+      # Parses query params out from full url string into hash.
+      #
+      # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group'
+      #       --> { title: 'Title', group: 'Group' }
+      def query_params(url)
+        strong_memoize(:query_params) do
+          Gitlab::Metrics::Dashboard::Url.parse_query(url)
+        end
+      end
     end
   end
 end

lib/banzai/filter/inline_metrics_filter.rb

@@ -42,14 +42,6 @@ module Banzai
           **query_params(params['url'])
         )
       end
-
-      # Parses query params out from full url string into hash.
-      #
-      # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group'
-      #       --> { title: 'Title', group: 'Group' }
-      def query_params(url)
-        Gitlab::Metrics::Dashboard::Url.parse_query(url)
-      end
     end
   end
 end

lib/gitlab/config_checker/puma_rugged_checker.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ConfigChecker
+    module PumaRuggedChecker
+      extend self
+      extend Gitlab::Git::RuggedImpl::UseRugged
+
+      def check
+        return [] unless Gitlab::Runtime.puma?
+
+        notices = []
+
+        link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html">'
+        link_end = '</a>'
+        notices << {
+          type: 'info',
+          message: _('You are running Puma, which is currently experimental. '\
+                   'More information is available in our '\
+                   '%{link_start}documentation%{link_end}.') % { link_start: link_start, link_end: link_end }
+        }
+
+        if running_puma_with_multiple_threads?
+          link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html">'
+          link_end = '</a>'
+          notices << {
+            type: 'info',
+            message: _('Puma is running with a thread count above 1. '\
+                     'Information on deprecated GitLab features in this configuration is available in the '\
+                     '%{link_start}documentation%{link_end}.') % { link_start: link_start, link_end: link_end }
+          }
+        end
+
+        if running_puma_with_multiple_threads? && rugged_enabled_through_feature_flag?
+          link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">'
+          link_end = '</a>'
+          notices << {
+            type: 'warning',
+            message: _('Puma is running with a thread count above 1 and the rugged '\
+                     'service is enabled. This may decrease performance in some environments. '\
+                     'See our %{link_start}documentation%{link_end} '\
+                     'for details of this issue.') % { link_start: link_start, link_end: link_end }
+          }
+        end
+
+        notices
+      end
+    end
+  end
+end

lib/gitlab/git/rugged_impl/use_rugged.rb

@@ -15,12 +15,6 @@ module Gitlab
           Gitlab::GitalyClient.can_use_disk?(repo.storage)
         end
 
-        def running_puma_with_multiple_threads?
-          return false unless Gitlab::Runtime.puma?
-
-          ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1
-        end
-
         def execute_rugged_call(method_name, *args)
           Gitlab::GitalyClient::StorageSettings.allow_disk_access do
             start = Gitlab::Metrics::System.monotonic_time
@@ -43,6 +37,22 @@ module Gitlab
             result
           end
         end
+
+        def running_puma_with_multiple_threads?
+          return false unless Gitlab::Runtime.puma?
+
+          ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1
+        end
+
+        def rugged_feature_keys
+          Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS
+        end
+
+        def rugged_enabled_through_feature_flag?
+          rugged_feature_keys.any? do |feature_key|
+            Feature.enabled?(feature_key)
+          end
+        end
       end
     end
   end

locale/gitlab.pot

@@ -15659,6 +15659,12 @@ msgstr ""
 msgid "Pull"
 msgstr ""
 
+msgid "Puma is running with a thread count above 1 and the rugged service is enabled. This may decrease performance in some environments. See our %{link_start}documentation%{link_end} for details of this issue."
+msgstr ""
+
+msgid "Puma is running with a thread count above 1. Information on deprecated GitLab features in this configuration is available in the %{link_start}documentation%{link_end}."
+msgstr ""
+
 msgid "Purchase more minutes"
 msgstr ""
 
@@ -22173,6 +22179,9 @@ msgstr ""
 msgid "You are receiving this message because you are a GitLab administrator for %{url}."
 msgstr ""
 
+msgid "You are running Puma, which is currently experimental. More information is available in our %{link_start}documentation%{link_end}."
+msgstr ""
+
 msgid "You can %{linkStart}view the blob%{linkEnd} instead."
 msgstr ""
 

scripts/lint-rugged

@@ -15,6 +15,9 @@ ALLOWED = [
   'lib/gitlab/git/rugged_impl/',
   'lib/gitlab/gitaly_client/storage_settings.rb',
 
+  # Needed to detect Rugged enabled: https://gitlab.com/gitlab-org/gitlab/issues/35371
+  'lib/gitlab/config_checker/puma_rugged_checker.rb',
+
   # Needed for logging
   'config/initializers/peek.rb',
   'config/initializers/lograge.rb',
@@ -27,7 +30,8 @@ ALLOWED = [
 rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines
 rugged_lines = rugged_lines.select { |l| /^[^:]*\.rb:/ =~ l }
 rugged_lines = rugged_lines.reject { |l| l.start_with?(*ALLOWED) }
-rugged_lines = rugged_lines.reject { |l| /(include|prepend) Gitlab::Git::RuggedImpl/ =~ l}
+rugged_lines = rugged_lines.reject { |l| /(include|prepend) Gitlab::Git::RuggedImpl/ =~ l }
+rugged_lines = rugged_lines.reject { |l| l.include?('Gitlab::ConfigChecker::PumaRuggedChecker.check') }
 rugged_lines = rugged_lines.reject do |line|
   code, _comment = line.split('# ', 2)
   code !~ /rugged/i

spec/lib/gitlab/config_checker/puma_rugged_checker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ConfigChecker::PumaRuggedChecker do
+  describe '#check' do
+    subject { described_class.check }
+
+    context 'application is not puma' do
+      before do
+        allow(Gitlab::Runtime).to receive(:puma?).and_return(false)
+      end
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'application is puma' do
+      let(:notice_running_puma) do
+        {
+            type: 'info',
+            message: 'You are running Puma, which is currently experimental. '\
+                     'More information is available in our '\
+                     '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html">documentation</a>.'
+        }
+      end
+      let(:notice_multi_threaded_puma) do
+        {
+            type: 'info',
+            message: 'Puma is running with a thread count above 1. '\
+                     'Information on deprecated GitLab features in this configuration is available in the '\
+                     '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html">documentation</a>.'\
+        }
+      end
+      let(:notice_multi_threaded_puma_with_rugged) do
+        {
+            type: 'warning',
+            message: 'Puma is running with a thread count above 1 and the rugged '\
+                     'service is enabled. This may decrease performance in some environments. '\
+                     'See our <a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">documentation</a> '\
+                     'for details of this issue.'
+        }
+      end
+
+      before do
+        allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
+        allow(described_class).to receive(:running_puma_with_multiple_threads?).and_return(multithreaded_puma)
+        allow(described_class).to receive(:rugged_enabled_through_feature_flag?).and_return(rugged_enabled)
+      end
+
+      context 'not multithreaded_puma and rugged API enabled' do
+        let(:multithreaded_puma) { false }
+        let(:rugged_enabled) { true }
+
+        it 'report running puma notice' do
+          is_expected.to contain_exactly(notice_running_puma)
+        end
+      end
+
+      context 'not multithreaded_puma and rugged API is not enabled' do
+        let(:multithreaded_puma) { false }
+        let(:rugged_enabled) { false }
+
+        it 'report running puma notice' do
+          is_expected.to contain_exactly(notice_running_puma)
+        end
+      end
+
+      context 'multithreaded_puma and rugged API is not enabled' do
+        let(:multithreaded_puma) { true }
+        let(:rugged_enabled) { false }
+
+        it 'report running puma notice and multi-thread puma notice' do
+          is_expected.to contain_exactly(notice_running_puma, notice_multi_threaded_puma)
+        end
+      end
+
+      context 'multithreaded_puma and rugged API is enabled' do
+        let(:multithreaded_puma) { true }
+        let(:rugged_enabled) { true }
+
+        it 'report puma/multi_threaded_puma/multi_threaded_puma_with_rugged notices' do
+          is_expected.to contain_exactly(notice_running_puma, notice_multi_threaded_puma, notice_multi_threaded_puma_with_rugged)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/git/rugged_impl/use_rugged_spec.rb

@@ -152,6 +152,43 @@ describe Gitlab::Git::RuggedImpl::UseRugged, :seed_helper do
     end
   end
 
+  describe '#rugged_enabled_through_feature_flag?' do
+    subject { wrapper.send(:rugged_enabled_through_feature_flag?) }
+
+    before do
+      allow(Feature).to receive(:enabled?).with(:feature_key_1).and_return(true)
+      allow(Feature).to receive(:enabled?).with(:feature_key_2).and_return(true)
+      allow(Feature).to receive(:enabled?).with(:feature_key_3).and_return(false)
+      allow(Feature).to receive(:enabled?).with(:feature_key_4).and_return(false)
+
+      stub_const('Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS', feature_keys)
+    end
+
+    context 'no feature keys given' do
+      let(:feature_keys) { [] }
+
+      it { is_expected.to be_falsey }
+    end
+
+    context 'all features are enabled' do
+      let(:feature_keys) { [:feature_key_1, :feature_key_2] }
+
+      it { is_expected.to be_truthy}
+    end
+
+    context 'all features are not enabled' do
+      let(:feature_keys) { [:feature_key_3, :feature_key_4] }
+
+      it { is_expected.to be_falsey }
+    end
+
+    context 'some feature is enabled' do
+      let(:feature_keys) { [:feature_key_4, :feature_key_2] }
+
+      it { is_expected.to be_truthy }
+    end
+  end
+
   def create_temporary_gitaly_metadata_file
     tmp = Tempfile.new('.gitaly-metadata')
     gitaly_metadata = {

spec/serializers/build_details_entity_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe BuildDetailsEntity do
   include ProjectForksHelper
 
-  set(:user) { create(:admin) }
+  let_it_be(:user) { create(:admin) }
 
   it 'inherits from JobEntity' do
     expect(described_class).to be < JobEntity

spec/serializers/container_repository_entity_spec.rb

@@ -3,14 +3,10 @@
 require 'spec_helper'
 
 describe ContainerRepositoryEntity do
-  let(:entity) do
-    described_class.new(repository, request: request)
-  end
-
-  set(:project) { create(:project) }
-  set(:user) { create(:user) }
-  set(:repository) { create(:container_repository, project: project) }
-
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:repository) { create(:container_repository, project: project) }
+  let(:entity) { described_class.new(repository, request: request) }
   let(:request) { double('request') }
 
   subject { entity.as_json }

spec/serializers/container_tag_entity_spec.rb

@@ -3,14 +3,10 @@
 require 'spec_helper'
 
 describe ContainerTagEntity do
-  let(:entity) do
-    described_class.new(tag, request: request)
-  end
-
-  set(:project) { create(:project) }
-  set(:user) { create(:user) }
-  set(:repository) { create(:container_repository, name: 'image', project: project) }
-
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:repository) { create(:container_repository, name: 'image', project: project) }
+  let(:entity) { described_class.new(tag, request: request) }
   let(:request) { double('request') }
   let(:tag) { repository.tag('test') }
 

spec/serializers/deployment_serializer_spec.rb

@@ -3,9 +3,8 @@
 require 'spec_helper'
 
 describe DeploymentSerializer do
-  set(:project) { create(:project, :repository) }
-  set(:user) { create(:user, email: project.commit.author_email) }
-
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user, email: project.commit.author_email) }
   let(:resource) { create(:deployment, project: project, sha: project.commit.id) }
   let(:serializer) { described_class.new(request) }
 

spec/serializers/merge_request_serializer_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 describe MergeRequestSerializer do
-  set(:user) { create(:user) }
-  set(:resource) { create(:merge_request, description: "Description") }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:resource) { create(:merge_request, description: "Description") }
 
   let(:json_entity) do
     described_class.new(current_user: user)

spec/serializers/namespace_basic_entity_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe NamespaceBasicEntity do
-  set(:group) { create(:group) }
+  let_it_be(:group) { create(:group) }
   let(:entity) do
     described_class.represent(group)
   end

spec/serializers/pipeline_details_entity_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe PipelineDetailsEntity do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
   let(:request) { double('request') }
 
   it 'inherrits from PipelineEntity' do

spec/serializers/pipeline_entity_spec.rb

@@ -5,9 +5,9 @@ require 'spec_helper'
 describe PipelineEntity do
   include Gitlab::Routing
 
-  set(:project) { create(:project) }
-  set(:user) { create(:user) }
-  set(:project) { create(:project) }
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project) }
   let(:request) { double('request') }
 
   before do

spec/serializers/pipeline_serializer_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 describe PipelineSerializer do
-  set(:project) { create(:project, :repository) }
-  set(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
 
   let(:serializer) do
     described_class.new(current_user: user, project: project)

spec/serializers/project_import_entity_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe ProjectImportEntity do
   include ImportHelper
 
-  set(:project) { create(:project, import_status: :started, import_source: 'namespace/project') }
+  let_it_be(:project) { create(:project, import_status: :started, import_source: 'namespace/project') }
   let(:provider_url) { 'https://provider.com' }
   let(:entity) { described_class.represent(project, provider_url: provider_url) }
 

spec/serializers/project_serializer_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe ProjectSerializer do
-  set(:project) { create(:project) }
+  let_it_be(:project) { create(:project) }
   let(:provider_url) { 'http://provider.com' }
 
   context 'when serializer option is :import' do

spec/tasks/gitlab/web_hook_rake_spec.rb

@@ -3,11 +3,10 @@
 require 'rake_helper'
 
 describe 'gitlab:web_hook namespace rake tasks' do
-  set(:group) { create(:group) }
-
-  set(:project1) { create(:project, namespace: group) }
-  set(:project2) { create(:project, namespace: group) }
-  set(:other_group_project) { create(:project) }
+  let_it_be(:group, refind: true) { create(:group) }
+  let_it_be(:project1, reload: true) { create(:project, namespace: group) }
+  let_it_be(:project2, reload: true) { create(:project, namespace: group) }
+  let_it_be(:other_group_project, reload: true) { create(:project) }
 
   let(:url) { 'http://example.com' }
   let(:hook_urls) { (project1.hooks + project2.hooks).map(&:url) }

spec/views/shared/milestones/_top.html.haml_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'shared/milestones/_top.html.haml' do
-  set(:group) { create(:group) }
+  let_it_be(:group) { create(:group) }
   let(:project) { create(:project, group: group) }
   let(:milestone) { create(:milestone, project: project) }
 

spec/workers/detect_repository_languages_worker_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe DetectRepositoryLanguagesWorker do
-  set(:project) { create(:project) }
+  let_it_be(:project) { create(:project) }
 
   subject { described_class.new }
 

spec/workers/expire_build_instance_artifacts_worker_spec.rb

@@ -31,7 +31,7 @@ describe ExpireBuildInstanceArtifactsWorker do
     end
 
     context 'with not yet expired artifacts' do
-      set(:build) do
+      let_it_be(:build) do
         create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days)
       end
 

spec/workers/expire_job_cache_worker_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe ExpireJobCacheWorker do
-  set(:pipeline) { create(:ci_empty_pipeline) }
+  let_it_be(:pipeline) { create(:ci_empty_pipeline) }
   let(:project) { pipeline.project }
 
   describe '#perform' do

spec/workers/mail_scheduler/notification_service_worker_spec.rb

@@ -6,7 +6,7 @@ describe MailScheduler::NotificationServiceWorker do
   let(:worker) { described_class.new }
   let(:method) { 'new_key' }
 
-  set(:key) { create(:personal_key) }
+  let_it_be(:key) { create(:personal_key) }
 
   def serialize(*args)
     ActiveJob::Arguments.serialize(args)

spec/workers/pipeline_schedule_worker_spec.rb

@@ -7,8 +7,8 @@ describe PipelineScheduleWorker do
 
   subject { described_class.new.perform }
 
-  set(:project) { create(:project, :repository) }
-  set(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
 
   let!(:pipeline_schedule) do
     create(:ci_pipeline_schedule, :nightly, project: project, owner: user)

spec/workers/remote_mirror_notification_worker_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 describe RemoteMirrorNotificationWorker, :mailer do
-  set(:project) { create(:project, :repository, :remote_mirror) }
-  set(:mirror) { project.remote_mirrors.first }
+  let_it_be(:project) { create(:project, :repository, :remote_mirror) }
+  let_it_be(:mirror) { project.remote_mirrors.first }
 
   describe '#execute' do
     it 'calls NotificationService#remote_mirror_update_failed when the mirror exists' do

spec/workers/run_pipeline_schedule_worker_spec.rb

@@ -4,9 +4,9 @@ require 'spec_helper'
 
 describe RunPipelineScheduleWorker do
   describe '#perform' do
-    set(:project) { create(:project) }
-    set(:user) { create(:user) }
-    set(:pipeline_schedule) { create(:ci_pipeline_schedule, :nightly, project: project ) }
+    let_it_be(:project) { create(:project) }
+    let_it_be(:user) { create(:user) }
+    let_it_be(:pipeline_schedule) { create(:ci_pipeline_schedule, :nightly, project: project ) }
     let(:worker) { described_class.new }
 
     context 'when a project not found' do

spec/workers/update_external_pull_requests_worker_spec.rb

@@ -4,8 +4,8 @@ require 'spec_helper'
 
 describe UpdateExternalPullRequestsWorker do
   describe '#perform' do
-    set(:project) { create(:project, import_source: 'tanuki/repository') }
-    set(:user) { create(:user) }
+    let_it_be(:project) { create(:project, import_source: 'tanuki/repository') }
+    let_it_be(:user) { create(:user) }
     let(:worker) { described_class.new }
 
     before do