Merge branch 'issue-241744-kroki-companion-containers' into 'master'

Followup: Enable/disable additional diagram formats using Kroki

See merge request gitlab-org/gitlab!49304

Gemfile

@@ -156,7 +156,7 @@ gem 'wikicloth', '0.8.1'
 gem 'asciidoctor', '~> 2.0.10'
 gem 'asciidoctor-include-ext', '~> 0.3.1', require: false
 gem 'asciidoctor-plantuml', '~> 0.0.12'
-gem 'asciidoctor-kroki', '~> 0.2.2', require: false
+gem 'asciidoctor-kroki', '~> 0.3.0', require: false
 gem 'rouge', '~> 3.26.0'
 gem 'truncato', '~> 0.7.11'
 gem 'bootstrap_form', '~> 4.2.0'

Gemfile.lock

@@ -84,7 +84,7 @@ GEM
     asciidoctor (2.0.12)
     asciidoctor-include-ext (0.3.1)
       asciidoctor (>= 1.5.6, < 3.0.0)
-    asciidoctor-kroki (0.2.2)
+    asciidoctor-kroki (0.3.0)
       asciidoctor (~> 2.0)
     asciidoctor-plantuml (0.0.12)
       asciidoctor (>= 1.5.6, < 3.0.0)
@@ -1291,7 +1291,7 @@ DEPENDENCIES
   asana (~> 0.10.3)
   asciidoctor (~> 2.0.10)
   asciidoctor-include-ext (~> 0.3.1)
-  asciidoctor-kroki (~> 0.2.2)
+  asciidoctor-kroki (~> 0.3.0)
   asciidoctor-plantuml (~> 0.0.12)
   atlassian-jwt (~> 0.2.0)
   attr_encrypted (~> 3.1.0)

app/controllers/admin/application_settings_controller.rb

@@ -238,6 +238,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       *::ApplicationSettingsHelper.visible_attributes,
       *::ApplicationSettingsHelper.external_authorization_service_attributes,
       *ApplicationSetting.repository_storages_weighted_attributes,
+      *ApplicationSetting.kroki_formats_attributes.keys.map { |key| "kroki_formats_#{key}".to_sym },
       :lets_encrypt_notification_email,
       :lets_encrypt_terms_of_service_accepted,
       :domain_denylist_file,

app/helpers/application_settings_helper.rb

@@ -26,6 +26,16 @@ module ApplicationSettingsHelper
     end
   end
 
+  def kroki_available_formats
+    ApplicationSetting.kroki_formats_attributes.map do |key, value|
+      {
+        name: "kroki_formats_#{key}",
+        label: value[:label],
+        value: @application_setting.kroki_formats[key] || false
+      }
+    end
+  end
+
   def storage_weights
     ApplicationSetting.repository_storages_weighted_attributes.map do |attribute|
       storage = attribute.to_s.delete_prefix('repository_storages_weighted_')
@@ -259,6 +269,7 @@ module ApplicationSettingsHelper
       :personal_access_token_prefix,
       :kroki_enabled,
       :kroki_url,
+      :kroki_formats,
       :plantuml_enabled,
       :plantuml_url,
       :polling_interval_multiplier,

app/models/application_setting.rb

@@ -29,6 +29,21 @@ class ApplicationSetting < ApplicationRecord
     @repository_storages_weighted_atributes ||= Gitlab.config.repositories.storages.keys.map { |k| "repository_storages_weighted_#{k}".to_sym }.freeze
   end
 
+  def self.kroki_formats_attributes
+    {
+      blockdiag: {
+        label: 'BlockDiag (includes BlockDiag, SeqDiag, ActDiag, NwDiag, PacketDiag and RackDiag)'
+      },
+      bpmn: {
+        label: 'BPMN'
+      },
+      excalidraw: {
+        label: 'Excalidraw'
+      }
+    }
+  end
+
+  store_accessor :kroki_formats, *ApplicationSetting.kroki_formats_attributes.keys, prefix: true
   store_accessor :repository_storages_weighted, *Gitlab.config.repositories.storages.keys, prefix: true
 
   # Include here so it can override methods from
@@ -54,6 +69,7 @@ class ApplicationSetting < ApplicationRecord
 
   default_value_for :id, 1
   default_value_for :repository_storages_weighted, {}
+  default_value_for :kroki_formats, {}
 
   chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds
 
@@ -135,6 +151,8 @@ class ApplicationSetting < ApplicationRecord
 
   validate :validate_kroki_url, if: :kroki_enabled
 
+  validates :kroki_formats, json_schema: { filename: 'application_setting_kroki_formats' }
+
   validates :plantuml_url,
             presence: true,
             if: :plantuml_enabled
@@ -570,6 +588,25 @@ class ApplicationSetting < ApplicationRecord
     end
   end
 
+  kroki_formats_attributes.keys.each do |key|
+    define_method :"kroki_formats_#{key}=" do |value|
+      super(::Gitlab::Utils.to_boolean(value))
+    end
+  end
+
+  def kroki_format_supported?(diagram_type)
+    case diagram_type
+    when 'excalidraw'
+      return kroki_formats_excalidraw
+    when 'bpmn'
+      return kroki_formats_bpmn
+    end
+
+    return kroki_formats_blockdiag if ::Gitlab::Kroki::BLOCKDIAG_FORMATS.include?(diagram_type)
+
+    ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES.include?(diagram_type)
+  end
+
   private
 
   def parsed_grafana_url

app/models/application_setting_implementation.rb

@@ -176,6 +176,7 @@ module ApplicationSettingImplementation
         container_registry_expiration_policies_worker_capacity: 0,
         kroki_enabled: false,
         kroki_url: nil,
+        kroki_formats: { blockdiag: false, bpmn: false, excalidraw: false },
         rate_limiting_response_text: nil
       }
     end

app/validators/json_schemas/application_setting_kroki_formats.json

+{
+  "description": "Kroki formats",
+  "type": "object",
+  "properties": {
+    "bpmn": { "type": "boolean" },
+    "excalidraw": { "type": "boolean" },
+    "blockdiag": { "type": "boolean" }
+  },
+  "additionalProperties": false
+}

app/views/admin/application_settings/_kroki.html.haml

@@ -21,5 +21,13 @@
           = f.text_field :kroki_url, class: 'form-control gl-form-input', placeholder: 'http://your-kroki-instance:8000'
           .form-text.text-muted
             = (_('When Kroki is enabled, GitLab sends diagrams to an instance of Kroki to display them as images. You can use the free public cloud instance %{kroki_public_url} or you can %{install_link} on your own infrastructure. Once you\'ve installed Kroki, make sure to update the server URL to point to your instance.') % { kroki_public_url: '<code>https://kroki.io</code>', install_link: link_to('install Kroki', 'https://docs.kroki.io/kroki/setup/install/', target: '_blank') }).html_safe
+        .form-group
+          = f.label :kroki_formats, 'Additional diagram formats', class: 'label-bold'
+          .form-text.text-muted
+            = (_('Using additional formats requires starting the companion containers. Make sure that all %{kroki_images} are running.') % { kroki_images: link_to('required containers', 'https://docs.kroki.io/kroki/setup/install/#_images', target: '_blank') }).html_safe
+          - kroki_available_formats.each do |format|
+            .form-check
+              = f.check_box format[:name], class: 'form-check-input'
+              = f.label format[:name], format[:label], class: 'form-check-label'
 
       = f.submit _('Save changes'), class: "btn gl-button btn-success"

changelogs/unreleased/241744-additional-formats-kroki.yml

+---
+title: "Enable/disable additional diagram formats on Kroki"
+merge_request: 49304
+author: Guillaume Grossetie
+type: added

db/migrate/20201120092000_add_kroki_formats_to_application_settings_table.rb

+# frozen_string_literal: true
+
+class AddKrokiFormatsToApplicationSettingsTable < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    change_table :application_settings do |t|
+      t.jsonb :kroki_formats, null: false, default: {}
+    end
+  end
+end

db/schema_migrations/20201120092000

+c8f837a5fe7a1959af41f19f93b6dd96d8907a476626f124876ee8b10b120b71
\ No newline at end of file

db/structure.sql

@@ -9398,6 +9398,7 @@ CREATE TABLE application_settings (
     keep_latest_artifact boolean DEFAULT true NOT NULL,
     notes_create_limit integer DEFAULT 300 NOT NULL,
     notes_create_limit_allowlist text[] DEFAULT '{}'::text[] NOT NULL,
+    kroki_formats jsonb DEFAULT '{}'::jsonb NOT NULL,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
     CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),

lib/gitlab/kroki.rb

@@ -13,9 +13,7 @@ module Gitlab
         packetdiag
         rackdiag
       ].freeze
-    # Diagrams that require a companion container are disabled for now
     DIAGRAMS_FORMATS = ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES
-                           .reject { |diagram_type| diagram_type == 'mermaid' || diagram_type == 'bpmn' || BLOCKDIAG_FORMATS.include?(diagram_type) }
     DIAGRAMS_FORMATS_WO_PLANTUML = DIAGRAMS_FORMATS
                                         .reject { |diagram_type| diagram_type == 'plantuml' }
 
@@ -28,10 +26,18 @@ module Gitlab
 
       # If PlantUML is enabled, PlantUML diagrams will be processed by the PlantUML server.
       # In other words, the PlantUML server has precedence over Kroki since both can process PlantUML diagrams.
-      if current_settings.plantuml_enabled
-        DIAGRAMS_FORMATS_WO_PLANTUML
-      else
-        DIAGRAMS_FORMATS
+      diagram_formats = if current_settings.plantuml_enabled
+                          DIAGRAMS_FORMATS_WO_PLANTUML
+                        else
+                          DIAGRAMS_FORMATS
+                        end
+
+      # No additional diagram formats
+      return diagram_formats unless current_settings.kroki_formats.present?
+
+      # Diagrams that require a companion container must be explicitly enabled from the settings
+      diagram_formats.select do |diagram_type|
+        current_settings.kroki_format_supported?(diagram_type)
       end
     end
   end

locale/gitlab.pot

@@ -32271,6 +32271,9 @@ msgstr ""
 msgid "Using %{code_start}::%{code_end} denotes a %{link_start}scoped label set%{link_end}"
 msgstr ""
 
+msgid "Using additional formats requires starting the companion containers. Make sure that all %{kroki_images} are running."
+msgstr ""
+
 msgid "Using required encryption strategy when encrypted field is missing!"
 msgstr ""
 

spec/controllers/admin/application_settings_controller_spec.rb

@@ -150,6 +150,13 @@ RSpec.describe Admin::ApplicationSettingsController do
       expect(ApplicationSetting.current.repository_storages_weighted_default).to eq(75)
     end
 
+    it 'updates kroki_formats setting' do
+      put :update, params: { application_setting: { kroki_formats_excalidraw: '1' } }
+
+      expect(response).to redirect_to(general_admin_application_settings_path)
+      expect(ApplicationSetting.current.kroki_formats_excalidraw).to eq(true)
+    end
+
     it "updates default_branch_name setting" do
       put :update, params: { application_setting: { default_branch_name: "example_branch_name" } }
 

spec/helpers/application_settings_helper_spec.rb

@@ -194,4 +194,33 @@ RSpec.describe ApplicationSettingsHelper do
       it { is_expected.to be false }
     end
   end
+
+  describe '.kroki_available_formats' do
+    let(:application_setting) { build(:application_setting) }
+
+    before do
+      helper.instance_variable_set(:@application_setting, application_setting)
+      stub_application_setting(kroki_formats: { 'blockdiag' => true, 'bpmn' => false, 'excalidraw' => false })
+    end
+
+    it 'returns available formats correctly' do
+      expect(helper.kroki_available_formats).to eq([
+                                             {
+                                               name: 'kroki_formats_blockdiag',
+                                               label: 'BlockDiag (includes BlockDiag, SeqDiag, ActDiag, NwDiag, PacketDiag and RackDiag)',
+                                               value: true
+                                             },
+                                             {
+                                               name: 'kroki_formats_bpmn',
+                                               label: 'BPMN',
+                                               value: false
+                                             },
+                                             {
+                                               name: 'kroki_formats_excalidraw',
+                                               label: 'Excalidraw',
+                                               value: false
+                                             }
+                                           ])
+    end
+  end
 end

spec/lib/gitlab/asciidoc_spec.rb

@@ -510,6 +510,73 @@ module Gitlab
 
           expect(render(input, context)).to include(output.strip)
         end
+
+        it 'does not convert a blockdiag diagram to image' do
+          input = <<~ADOC
+            [blockdiag]
+            ....
+            blockdiag {
+              Kroki -> generates -> "Block diagrams";
+              Kroki -> is -> "very easy!";
+
+              Kroki [color = "greenyellow"];
+              "Block diagrams" [color = "pink"];
+              "very easy!" [color = "orange"];
+            }
+            ....
+          ADOC
+
+          output = <<~HTML
+            <div>
+            <div>
+            <pre>blockdiag {
+              Kroki -&gt; generates -&gt; "Block diagrams";
+              Kroki -&gt; is -&gt; "very easy!";
+
+              Kroki [color = "greenyellow"];
+              "Block diagrams" [color = "pink"];
+              "very easy!" [color = "orange"];
+            }</pre>
+            </div>
+            </div>
+          HTML
+
+          expect(render(input, context)).to include(output.strip)
+        end
+      end
+
+      context 'with Kroki and BlockDiag (additional format) enabled' do
+        before do
+          allow_any_instance_of(ApplicationSetting).to receive(:kroki_enabled).and_return(true)
+          allow_any_instance_of(ApplicationSetting).to receive(:kroki_url).and_return('https://kroki.io')
+          allow_any_instance_of(ApplicationSetting).to receive(:kroki_formats_blockdiag).and_return(true)
+        end
+
+        it 'converts a blockdiag diagram to image' do
+          input = <<~ADOC
+            [blockdiag]
+            ....
+            blockdiag {
+              Kroki -> generates -> "Block diagrams";
+              Kroki -> is -> "very easy!";
+
+              Kroki [color = "greenyellow"];
+              "Block diagrams" [color = "pink"];
+              "very easy!" [color = "orange"];
+            }
+            ....
+          ADOC
+
+          output = <<~HTML
+            <div>
+            <div>
+            <a class="no-attachment-icon" href="https://kroki.io/blockdiag/svg/eNpdzDEKQjEQhOHeU4zpPYFoYesRxGJ9bwghMSsbUYJ4d10UCZbDfPynolOek0Q8FsDeNCestoisNLmy-Qg7R3Blcm5hPcr0ITdaB6X15fv-_YdJixo2CNHI2lmK3sPRA__RwV5SzV80ZAegJjXSyfMFptc71w==" target="_blank" rel="noopener noreferrer"><img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Diagram" class="lazy" data-src="https://kroki.io/blockdiag/svg/eNpdzDEKQjEQhOHeU4zpPYFoYesRxGJ9bwghMSsbUYJ4d10UCZbDfPynolOek0Q8FsDeNCestoisNLmy-Qg7R3Blcm5hPcr0ITdaB6X15fv-_YdJixo2CNHI2lmK3sPRA__RwV5SzV80ZAegJjXSyfMFptc71w=="></a>
+            </div>
+            </div>
+          HTML
+
+          expect(render(input, context)).to include(output.strip)
+        end
       end
     end
 

spec/models/application_setting_spec.rb

@@ -964,6 +964,50 @@ RSpec.describe ApplicationSetting do
     end
   end
 
+  describe 'kroki_format_supported?' do
+    it 'returns true when Excalidraw is enabled' do
+      subject.kroki_formats_excalidraw = true
+      expect(subject.kroki_format_supported?('excalidraw')).to eq(true)
+    end
+
+    it 'returns true when BlockDiag is enabled' do
+      subject.kroki_formats_blockdiag = true
+      # format "blockdiag" aggregates multiple diagram types: actdiag, blockdiag, nwdiag...
+      expect(subject.kroki_format_supported?('actdiag')).to eq(true)
+      expect(subject.kroki_format_supported?('blockdiag')).to eq(true)
+    end
+
+    it 'returns false when BlockDiag is disabled' do
+      subject.kroki_formats_blockdiag = false
+      # format "blockdiag" aggregates multiple diagram types: actdiag, blockdiag, nwdiag...
+      expect(subject.kroki_format_supported?('actdiag')).to eq(false)
+      expect(subject.kroki_format_supported?('blockdiag')).to eq(false)
+    end
+
+    it 'returns false when the diagram type is optional and not enabled' do
+      expect(subject.kroki_format_supported?('bpmn')).to eq(false)
+    end
+
+    it 'returns true when the diagram type is enabled by default' do
+      expect(subject.kroki_format_supported?('vegalite')).to eq(true)
+      expect(subject.kroki_format_supported?('nomnoml')).to eq(true)
+      expect(subject.kroki_format_supported?('unknown-diagram-type')).to eq(false)
+    end
+
+    it 'returns false when the diagram type is unknown' do
+      expect(subject.kroki_format_supported?('unknown-diagram-type')).to eq(false)
+    end
+  end
+
+  describe 'kroki_formats' do
+    it 'returns the value for kroki_formats' do
+      subject.kroki_formats = { blockdiag: true, bpmn: false, excalidraw: true }
+      expect(subject.kroki_formats_blockdiag).to eq(true)
+      expect(subject.kroki_formats_bpmn).to eq(false)
+      expect(subject.kroki_formats_excalidraw).to eq(true)
+    end
+  end
+
   it 'does not allow to set weight for non existing storage' do
     setting.repository_storages_weighted = { invalid_storage: 100 }