Use Gitlab::Markdown for Asciidoc and ReferenceExtractor pipelines

app/helpers/gitlab_markdown_helper.rb

@@ -65,13 +65,16 @@ module GitlabMarkdownHelper
   end
 
   def asciidoc(text)
-    Gitlab::Asciidoc.render(text, {
-      commit: @commit,
-      project: @project,
-      project_wiki: @project_wiki,
+    Gitlab::Asciidoc.render(text,
+      project:      @project,
+      current_user: (current_user if defined?(current_user)),
+
+      # RelativeLinkFilter
+      project_wiki:   @project_wiki,
       requested_path: @path,
-      ref: @ref
-    })
+      ref:            @ref,
+      commit:         @commit
+    )
   end
 
   # Return the first line of +text+, up to +max_chars+, after parsing the line

lib/gitlab/asciidoc.rb

 require 'asciidoctor'
-require 'html/pipeline'
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
   # the resulting HTML through HTML pipeline filters.
   module Asciidoc
 
-    # Provide autoload paths for filters to prevent a circular dependency error
-    autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter'
-
     DEFAULT_ADOC_ATTRS = [
       'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab',
       'env-gitlab', 'source-highlighter=html-pipeline'
@@ -24,13 +20,11 @@ module Gitlab
     #                 :requested_path
     #                 :ref
     # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter
-    # html_opts     - a Hash of options for HTML output:
-    #                 :xhtml - output XHTML instead of HTML
     #
-    def self.render(input, context, asciidoc_opts = {}, html_opts = {})
-      asciidoc_opts = asciidoc_opts.reverse_merge(
+    def self.render(input, context, asciidoc_opts = {})
+      asciidoc_opts.reverse_merge!(
         safe: :secure,
-        backend: html_opts[:xhtml] ? :xhtml5 : :html5,
+        backend: :html5,
         attributes: []
       )
       asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
@@ -38,23 +32,10 @@ module Gitlab
       html = ::Asciidoctor.convert(input, asciidoc_opts)
 
       if context[:project]
-        result = HTML::Pipeline.new(filters).call(html, context)
-
-        save_opts = html_opts[:xhtml] ?
-          Nokogiri::XML::Node::SaveOptions::AS_XHTML : 0
-
-        html = result[:output].to_html(save_with: save_opts)
+        html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc))
       end
 
       html.html_safe
     end
-
-    private
-
-    def self.filters
-      [
-        Gitlab::Markdown::RelativeLinkFilter
-      ]
-    end
   end
 end

lib/gitlab/markdown.rb

@@ -21,9 +21,9 @@ module Gitlab
     # Returns an HTML-safe String
     def self.render(text, context = {})
       cache_key = context.delete(:cache_key)
+      cache_key = full_cache_key(cache_key, context[:pipeline])
 
       if cache_key
-        cache_key = full_cache_key(cache_key, context[:pipeline])
         Rails.cache.fetch(cache_key) do
           cacheless_render(text, context)
         end
@@ -32,25 +32,21 @@ module Gitlab
       end
     end
 
-    # Provide autoload paths for filters to prevent a circular dependency error
-    autoload :AutolinkFilter,               'gitlab/markdown/autolink_filter'
-    autoload :CommitRangeReferenceFilter,   'gitlab/markdown/commit_range_reference_filter'
-    autoload :CommitReferenceFilter,        'gitlab/markdown/commit_reference_filter'
-    autoload :EmojiFilter,                  'gitlab/markdown/emoji_filter'
-    autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter'
-    autoload :ExternalLinkFilter,           'gitlab/markdown/external_link_filter'
-    autoload :IssueReferenceFilter,         'gitlab/markdown/issue_reference_filter'
-    autoload :LabelReferenceFilter,         'gitlab/markdown/label_reference_filter'
-    autoload :MarkdownFilter,               'gitlab/markdown/markdown_filter'
-    autoload :MergeRequestReferenceFilter,  'gitlab/markdown/merge_request_reference_filter'
-    autoload :RedactorFilter,               'gitlab/markdown/redactor_filter'
-    autoload :RelativeLinkFilter,           'gitlab/markdown/relative_link_filter'
-    autoload :SanitizationFilter,           'gitlab/markdown/sanitization_filter'
-    autoload :SnippetReferenceFilter,       'gitlab/markdown/snippet_reference_filter'
-    autoload :SyntaxHighlightFilter,        'gitlab/markdown/syntax_highlight_filter'
-    autoload :TableOfContentsFilter,        'gitlab/markdown/table_of_contents_filter'
-    autoload :TaskListFilter,               'gitlab/markdown/task_list_filter'
-    autoload :UserReferenceFilter,          'gitlab/markdown/user_reference_filter'
+    def self.render_result(text, context = {})
+      pipeline = context[:pipeline] || :full
+
+      html_pipeline = html_pipelines[pipeline]
+
+      transformers = context_transformers[pipeline]
+      context = transformers.reduce(context) { |context, transformer| transformer.call(context) }
+
+      html_pipeline.call(text, context)
+    end
+
+    def self.cached?(cache_key, pipeline: :full)
+      cache_key = full_cache_key(cache_key, pipeline)
+      cache_key ? Rails.cache.exist?(cache_key) : false
+    end
 
     # Perform post-processing on an HTML String
     #
@@ -66,21 +62,39 @@ module Gitlab
     #
     # Returns an HTML-safe String
     def self.post_process(html, context)
-      doc = html_pipelines[:post_process].to_document(html, context)
+      html_pipeline = html_pipelines[:post_process]
 
       if context[:xhtml]
-        doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
+        html_pipeline.to_document(html, context).to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
       else
-        doc.to_html
+        html_pipeline.to_html(html, context)
       end.html_safe
     end
 
     private
-    FILTERS = {
-      plain_markdown: [
-        Gitlab::Markdown::MarkdownFilter
-      ],
-      gfm: [
+
+    # Provide autoload paths for filters to prevent a circular dependency error
+    autoload :AutolinkFilter,               'gitlab/markdown/autolink_filter'
+    autoload :CommitRangeReferenceFilter,   'gitlab/markdown/commit_range_reference_filter'
+    autoload :CommitReferenceFilter,        'gitlab/markdown/commit_reference_filter'
+    autoload :EmojiFilter,                  'gitlab/markdown/emoji_filter'
+    autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter'
+    autoload :ExternalLinkFilter,           'gitlab/markdown/external_link_filter'
+    autoload :IssueReferenceFilter,         'gitlab/markdown/issue_reference_filter'
+    autoload :LabelReferenceFilter,         'gitlab/markdown/label_reference_filter'
+    autoload :MarkdownFilter,               'gitlab/markdown/markdown_filter'
+    autoload :MergeRequestReferenceFilter,  'gitlab/markdown/merge_request_reference_filter'
+    autoload :RedactorFilter,               'gitlab/markdown/redactor_filter'
+    autoload :RelativeLinkFilter,           'gitlab/markdown/relative_link_filter'
+    autoload :SanitizationFilter,           'gitlab/markdown/sanitization_filter'
+    autoload :SnippetReferenceFilter,       'gitlab/markdown/snippet_reference_filter'
+    autoload :SyntaxHighlightFilter,        'gitlab/markdown/syntax_highlight_filter'
+    autoload :TableOfContentsFilter,        'gitlab/markdown/table_of_contents_filter'
+    autoload :TaskListFilter,               'gitlab/markdown/task_list_filter'
+    autoload :UserReferenceFilter,          'gitlab/markdown/user_reference_filter'
+
+    def self.gfm_filters
+      @gfm_filters ||= [
         Gitlab::Markdown::SyntaxHighlightFilter,
         Gitlab::Markdown::SanitizationFilter,
 
@@ -99,71 +113,99 @@ module Gitlab
         Gitlab::Markdown::LabelReferenceFilter,
 
         Gitlab::Markdown::TaskListFilter
-      ],
+      ]
+    end
 
-      full:           [:plain_markdown, :gfm],
-      atom:           :full,
-      email:          :full,
-      description:    :full,
-      single_line:    :gfm,
+    def self.all_filters
+      @all_filters ||= {
+        plain_markdown: [
+          Gitlab::Markdown::MarkdownFilter
+        ],
+        gfm: gfm_filters,
+
+        full:           [:plain_markdown, :gfm],
+        atom:           :full,
+        email:          :full,
+        description:    :full,
+        single_line:    :gfm,
+
+        asciidoc: [
+          Gitlab::Markdown::RelativeLinkFilter
+        ],
+
+        post_process: [
+          Gitlab::Markdown::RelativeLinkFilter, 
+          Gitlab::Markdown::RedactorFilter
+        ],
+
+        reference_extraction: [
+          Gitlab::Markdown::ReferenceGathererFilter
+        ]
+      }
+    end
 
-      post_process: [
-        Gitlab::Markdown::RelativeLinkFilter, 
-        Gitlab::Markdown::RedactorFilter
-      ]
-    }
-
-    CONTEXT_TRANSFORMERS = {
-      gfm: {
-        only_path: true,
-
-        # EmojiFilter
-        asset_host: Gitlab::Application.config.asset_host,
-        asset_root: Gitlab.config.gitlab.base_url
-      },
-      full: :gfm,
-
-      atom: [
-        :full, 
-        { 
-          only_path: false, 
-          xhtml: true 
-        }
-      ],
-      email: [
-        :full,
-        { only_path: false }
-      ],
-      description: [
-        :full,
-        { 
-          # SanitizationFilter
-          inline_sanitization: true
+    def self.all_context_transformers
+      @all_context_transformers ||= {
+        gfm: {
+          only_path: true,
+
+          # EmojiFilter
+          asset_host: Gitlab::Application.config.asset_host,
+          asset_root: Gitlab.config.gitlab.base_url
+        },
+        full: :gfm,
+
+        atom: [
+          :full, 
+          { 
+            only_path: false, 
+            xhtml: true 
+          }
+        ],
+        email: [
+          :full,
+          { 
+            only_path: false
+          }
+        ],
+        description: [
+          :full,
+          { 
+            # SanitizationFilter
+            inline_sanitization: true
+          }
+        ],
+        single_line: :gfm,
+
+        post_process: {
+          post_process: true
         }
-      ],
-      single_line: :gfm,
-
-      post_process: {
-        post_process: true
       }
-    }
+    end
+
+    def self.html_filters
+      @html_filters ||= Hash.new do |hash, pipeline|
+        filters = get_filters(pipeline)
+        hash[pipeline] = filters if pipeline.is_a?(Symbol)
+        filters
+      end
+    end
 
     def self.html_pipelines
       @html_pipelines ||= Hash.new do |hash, pipeline|
         filters = get_filters(pipeline)
-        HTML::Pipeline.new(filters)
+        html_pipeline = HTML::Pipeline.new(filters)
+        hash[pipeline] = html_pipeline if pipeline.is_a?(Symbol)
+        html_pipeline
       end
     end
 
-    def self.cacheless_render(text, context = {})
-      pipeline = context[:pipeline] || :full
-
-      html_pipeline = html_pipelines[pipeline]
-
-      transformers = get_context_transformers(pipeline)
-      context = transformers.reduce(context) { |context, transformer| transformer.call(context) }
-
-      html_pipeline.to_html(text, context)
+    def self.context_transformers
+      @context_transformers ||= Hash.new do |hash, pipeline|
+        transformers = get_context_transformers(pipeline)
+        hash[pipeline] = transformers if pipeline.is_a?(Symbol)
+        transformers
+      end
     end
 
     def self.get_filters(pipelines)
@@ -172,9 +214,9 @@ module Gitlab
         when Class
           pipeline
         when Symbol
-          get_filters(FILTERS[pipeline])
+          html_filters[all_filters[pipeline]]
         when Array
-          get_filters(pipeline)
+          html_filters[pipeline]
         end
       end.compact
     end
@@ -187,14 +229,26 @@ module Gitlab
         when Proc
           pipeline
         when Symbol
-          get_context_transformers(CONTEXT_TRANSFORMERS[pipeline])
+          context_transformers[all_context_transformers[pipeline]]
         when Array
-          get_context_transformers(pipeline)
+          context_transformers[pipeline]
         end
       end.compact
     end
 
+    def self.cacheless_render(text, context = {})
+      result = render_result(text, context)
+      output = result[:output]
+      if output.respond_to?(:to_html)
+        output.to_html
+      else
+        output.to_s
+      end
+    end
+
     def self.full_cache_key(cache_key, pipeline = :full)
+      return unless cache_key && pipeline.is_a?(Symbol)
+
       ["markdown", *cache_key, pipeline]
     end
   end

lib/gitlab/reference_extractor.rb

@@ -14,7 +14,8 @@ module Gitlab
     def analyze(text, cache_key: nil)
       references.clear
 
-      @document = Gitlab::Markdown.render(text, project: project, cache_key: cache_key)
+      @pipeline = Gitlab::Markdown.cached?(cache_key, pipeline: :full) ? :full : :plain_markdown
+      @html = Gitlab::Markdown.render(text, project: project, cache_key: cache_key, pipeline: @pipeline)
     end
 
     %i(user label issue merge_request snippet commit commit_range).each do |type|
@@ -45,14 +46,19 @@ module Gitlab
       filter = Gitlab::Markdown.const_get(klass)
 
       context = {
-        project:              project,
-        current_user:         current_user,
+        pipeline: [:reference_extraction],
+
+        project:      project,
+        current_user: current_user,
+
+        # ReferenceGathererFilter
         load_lazy_references: false,
         reference_filter:     filter
       }
 
-      result = self.class.pipeline.call(@document, context)
+      context[:pipeline].unshift(filter) unless @pipeline == :full
 
+      result = Gitlab::Markdown.render_result(@html, context)
       values = result[:references][filter_type].uniq
 
       if @load_lazy_references
@@ -61,9 +67,5 @@ module Gitlab
 
       values
     end
-
-    def self.pipeline
-      @pipeline ||= HTML::Pipeline.new([Gitlab::Markdown::ReferenceGathererFilter])
-    end
   end
 end

spec/lib/gitlab/asciidoc_spec.rb

@@ -50,9 +50,9 @@ module Gitlab
         filtered_html = '<b>ASCII</b>'
 
         allow(Asciidoctor).to receive(:convert).and_return(html)
-        expect_any_instance_of(HTML::Pipeline).to receive(:call)
-          .with(html, context)
-          .and_return(output: Nokogiri::HTML.fragment(filtered_html))
+        expect(Gitlab::Markdown).to receive(:render)
+          .with(html, context.merge(pipeline: :asciidoc))
+          .and_return(filtered_html)
 
         expect( render('foo', context) ).to eql filtered_html
       end