Add latest changes from gitlab-org/gitlab@13-9-stable-ee

GITLAB_SHELL_VERSION

-13.16.1
+13.17.0

Gemfile

@@ -156,7 +156,7 @@ gem 'wikicloth', '0.8.1'
 gem 'asciidoctor', '~> 2.0.10'
 gem 'asciidoctor-include-ext', '~> 0.3.1', require: false
 gem 'asciidoctor-plantuml', '~> 0.0.12'
-gem 'asciidoctor-kroki', '~> 0.3.0', require: false
+gem 'asciidoctor-kroki', '~> 0.4.0', require: false
 gem 'rouge', '~> 3.26.0'
 gem 'truncato', '~> 0.7.11'
 gem 'bootstrap_form', '~> 4.2.0'

Gemfile.lock

@@ -84,7 +84,7 @@ GEM
     asciidoctor (2.0.12)
     asciidoctor-include-ext (0.3.1)
       asciidoctor (>= 1.5.6, < 3.0.0)
-    asciidoctor-kroki (0.3.0)
+    asciidoctor-kroki (0.4.0)
       asciidoctor (~> 2.0)
     asciidoctor-plantuml (0.0.12)
       asciidoctor (>= 1.5.6, < 3.0.0)
@@ -1291,7 +1291,7 @@ DEPENDENCIES
   asana (~> 0.10.3)
   asciidoctor (~> 2.0.10)
   asciidoctor-include-ext (~> 0.3.1)
-  asciidoctor-kroki (~> 0.3.0)
+  asciidoctor-kroki (~> 0.4.0)
   asciidoctor-plantuml (~> 0.0.12)
   atlassian-jwt (~> 0.2.0)
   attr_encrypted (~> 3.1.0)

app/models/application_setting.rb

@@ -58,8 +58,9 @@ class ApplicationSetting < ApplicationRecord
   serialize :domain_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
   serialize :domain_denylist, Array # rubocop:disable Cop/ActiveRecordSerialize
   serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize
-  serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
+
   # See https://gitlab.com/gitlab-org/gitlab/-/issues/300916
+  serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
   serialize :asset_proxy_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize
 
   cache_markdown_field :sign_in_text

app/models/application_setting_implementation.rb

@@ -280,13 +280,18 @@ module ApplicationSettingImplementation
     self.notes_create_limit_allowlist = strings_to_array(values).map(&:downcase)
   end
 
-  def asset_proxy_allowlist=(values)
+  def asset_proxy_whitelist=(values)
     values = strings_to_array(values) if values.is_a?(String)
 
     # make sure we always allow the running host
     values << Gitlab.config.gitlab.host unless values.include?(Gitlab.config.gitlab.host)
 
-    self[:asset_proxy_allowlist] = values
+    self[:asset_proxy_whitelist] = values
+  end
+  alias_method :asset_proxy_allowlist=, :asset_proxy_whitelist=
+
+  def asset_proxy_allowlist
+    read_attribute(:asset_proxy_whitelist)
   end
 
   def repository_storages

app/services/application_settings/update_service.rb

@@ -6,7 +6,7 @@ module ApplicationSettings
 
     attr_reader :params, :application_setting
 
-    MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_allowlist).freeze
+    MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_whitelist).freeze
 
     def execute
       result = update_settings

changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml

+---
+title: Update Kroki to fix Wavedrom graphs
+merge_request: 55659
+author:
+type: fixed

changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml

+---
+title: Rename asset_proxy_allowlist column
+merge_request: 55884
+author:
+type: fixed

changelogs/unreleased/sh-bump-gitlab-shell-version.yml

+---
+title: Upgrade gitlab-shell to v13.17.0
+merge_request: 55295
+author:
+type: fixed

changelogs/unreleased/sh-fix-kroki-format-filtering.yml

+---
+title: Fix disabling of Kroki optional formats
+merge_request: 55665
+author:
+type: fixed

db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb

+# frozen_string_literal: true
+
+class RenameAssetProxyAllowlistOnApplicationSettings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers::V2
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :application_settings,
+      :asset_proxy_allowlist,
+      :asset_proxy_whitelist
+  end
+
+  def down
+    undo_rename_column_concurrently :application_settings,
+      :asset_proxy_allowlist,
+      :asset_proxy_whitelist
+  end
+end

db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb

+# frozen_string_literal: true
+
+class CleanUpAssetProxyAllowlistRenameOnApplicationSettings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers::V2
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :application_settings,
+      :asset_proxy_allowlist,
+      :asset_proxy_whitelist
+  end
+
+  def down
+    undo_cleanup_concurrent_column_rename :application_settings,
+      :asset_proxy_allowlist,
+      :asset_proxy_whitelist
+  end
+end

db/schema_migrations/20210301200601

+21ae7ea7cbf1d34c7b9dc300a641eaf975ed1e33f5bc519494cd37c4a661bec8
\ No newline at end of file

db/schema_migrations/20210301200959

+28b90c9b7c2e4f2e2b12088f5aee85c16dfb567f89ed6a8e771f2c5d91c818d9
\ No newline at end of file

db/structure.sql

@@ -9394,11 +9394,11 @@ CREATE TABLE application_settings (
     container_registry_cleanup_tags_service_max_list_size integer DEFAULT 200 NOT NULL,
     enforce_ssh_key_expiration boolean DEFAULT false NOT NULL,
     git_two_factor_session_expiry integer DEFAULT 15 NOT NULL,
-    asset_proxy_allowlist text,
     keep_latest_artifact boolean DEFAULT true NOT NULL,
     notes_create_limit integer DEFAULT 300 NOT NULL,
     notes_create_limit_allowlist text[] DEFAULT '{}'::text[] NOT NULL,
     kroki_formats jsonb DEFAULT '{}'::jsonb NOT NULL,
+    asset_proxy_whitelist text,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
     CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),

doc/administration/object_storage.md

@@ -31,6 +31,8 @@ GitLab has been tested on a number of object storage providers:
   HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806).
   Be sure to upgrade to GitLab v13.3.0 or above if you use S3 storage with this hardware.
 
+- Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying).
+
 ## Configuration guides
 
 There are two ways of specifying object storage configuration in GitLab:
@@ -752,7 +754,6 @@ To set up an instance profile:
                "Action": [
                    "s3:PutObject",
                    "s3:GetObject",
-                   "s3:AbortMultipartUpload",
                    "s3:DeleteObject"
                ],
                "Resource": "arn:aws:s3:::test-bucket/*"
@@ -764,3 +765,18 @@ To set up an instance profile:
 1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/)
    to the EC2 instance hosting your GitLab instance.
 1. Configure GitLab to use it via the `use_iam_profile` configuration option.
+
+### Multi-threaded copying
+
+GitLab uses the [S3 Upload Part Copy API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html)
+to accelerate the copying of files within a bucket. Ceph S3 [prior to Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/)
+does not support this and [returns a 404 error when files are copied during the upload process](https://gitlab.com/gitlab-org/gitlab/-/issues/300604).
+
+The feature can be disabled using the `:s3_multithreaded_uploads`
+feature flag. To disable the feature, ask a GitLab administrator with
+[Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags)
+to run the following command:
+
+```ruby
+Feature.disable(:s3_multithreaded_uploads)
+```

lib/gitlab/background_migration/backfill_artifact_expiry_date.rb

@@ -6,9 +6,9 @@ module Gitlab
     class BackfillArtifactExpiryDate
       include Gitlab::Utils::StrongMemoize
 
-      BATCH_SIZE = 1_000
-      DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze
+      SWITCH_DATE = Date.new(2020, 06, 22).freeze
       OLD_ARTIFACT_AGE = 15.months
+      BATCH_SIZE = 1_000
       OLD_ARTIFACT_EXPIRY_OFFSET = 3.months
       RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year
 
@@ -18,16 +18,17 @@ module Gitlab
 
         self.table_name = 'ci_job_artifacts'
 
-        scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
-        scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) }
         scope :without_expiry_date, -> { where(expire_at: nil) }
+        scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) }
+        scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
         scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) }
         scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) }
       end
 
       def perform(start_id, end_id)
-        Ci::JobArtifact.between(start_id, end_id)
-          .without_expiry_date.before_default_expiration_switch
+        Ci::JobArtifact
+          .without_expiry_date.before_switch
+          .between(start_id, end_id)
           .each_batch(of: BATCH_SIZE) do |batch|
           batch.old.update_all(expire_at: old_artifact_expiry_date)
           batch.recent.update_all(expire_at: recent_artifact_expiry_date)

lib/gitlab/kroki.rb

@@ -31,9 +31,6 @@ module Gitlab
                           DIAGRAMS_FORMATS
                         end
 
-      # No additional diagram formats
-      return diagram_formats unless current_settings.kroki_formats.present?
-
       # Diagrams that require a companion container must be explicitly enabled from the settings
       diagram_formats.select do |diagram_type|
         current_settings.kroki_format_supported?(diagram_type)

spec/deprecation_toolkit_env.rb

@@ -61,6 +61,7 @@ module DeprecationToolkitEnv
       batch-loader-1.4.0/lib/batch_loader/graphql.rb
       carrierwave-1.3.1/lib/carrierwave/sanitized_file.rb
       activerecord-6.0.3.4/lib/active_record/relation.rb
+      asciidoctor-2.0.12/lib/asciidoctor/extensions.rb
     ]
   end
 

spec/lib/gitlab/kroki_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kroki do
+  using RSpec::Parameterized::TableSyntax
+
+  describe '.formats' do
+    def default_formats
+      %w[bytefield c4plantuml ditaa erd graphviz nomnoml plantuml svgbob umlet vega vegalite wavedrom].freeze
+    end
+
+    subject { described_class.formats(Gitlab::CurrentSettings) }
+
+    where(:enabled_formats, :expected_formats) do
+      ''           | default_formats
+      'blockdiag'  | default_formats + %w[actdiag blockdiag nwdiag packetdiag rackdiag seqdiag]
+      'bpmn'       | default_formats + %w[bpmn]
+      'excalidraw' | default_formats + %w[excalidraw]
+    end
+
+    with_them do
+      before do
+        kroki_formats =
+          if enabled_formats.present?
+            { enabled_formats => true }
+          else
+            {}
+          end
+
+        stub_application_setting(kroki_enabled: true, kroki_url: "http://localhost:8000", kroki_formats: kroki_formats)
+      end
+
+      it 'returns the expected formats' do
+        expect(subject).to match_array(expected_formats)
+      end
+    end
+  end
+end

spec/models/application_setting_spec.rb

@@ -650,6 +650,32 @@ RSpec.describe ApplicationSetting do
         end
       end
 
+      describe '#asset_proxy_whitelist' do
+        context 'when given an Array' do
+          it 'sets the domains and adds current running host' do
+            setting.asset_proxy_whitelist = ['example.com', 'assets.example.com']
+            expect(setting.asset_proxy_whitelist).to eq(['example.com', 'assets.example.com', 'localhost'])
+          end
+        end
+
+        context 'when given a String' do
+          it 'sets multiple domains with spaces' do
+            setting.asset_proxy_whitelist = 'example.com *.example.com'
+            expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+          end
+
+          it 'sets multiple domains with newlines and a space' do
+            setting.asset_proxy_whitelist = "example.com\n *.example.com"
+            expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+          end
+
+          it 'sets multiple domains with commas' do
+            setting.asset_proxy_whitelist = "example.com, *.example.com"
+            expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+          end
+        end
+      end
+
       describe '#asset_proxy_allowlist' do
         context 'when given an Array' do
           it 'sets the domains and adds current running host' do

spec/services/application_settings/update_service_spec.rb

@@ -123,6 +123,7 @@ RSpec.describe ApplicationSettings::UpdateService do
     it_behaves_like 'invalidates markdown cache', { asset_proxy_url: 'http://test.com' }
     it_behaves_like 'invalidates markdown cache', { asset_proxy_secret_key: 'another secret' }
     it_behaves_like 'invalidates markdown cache', { asset_proxy_allowlist: ['domain.com'] }
+    it_behaves_like 'invalidates markdown cache', { asset_proxy_whitelist: ['domain.com'] }
 
     context 'when also setting the local_markdown_version' do
       let(:params) { { asset_proxy_enabled: true, local_markdown_version: 12 } }