Commit 5ffed49f authored by Robert Speicher's avatar Robert Speicher

Merge branch 'lazy-reference-extractor' into 'master'

Move Markdown/reference logic from `Gitlab::Markdown` to `Banzai`

- Moves from `Gitlab::Markdown` to `Banzai`
- Moves filters and pipelines into their own namespace: `Banzai::Filter` and `Banzai::Pipeline`
- No more `autoload`!
- Split up `Gitlab::ReferenceExtractor` into `Banzai::ReferenceExtractor` and `Gitlab::ReferenceExtractor`
- Replace `something(load_lazy_references: true)` by `Gitlab::ReferenceExtractor.lazily { something }`

Goes from:

```ruby
def referenced_merge_requests
  references = [self, *notes].flat_map do |note|
    note.all_references(load_lazy_references: false).merge_requests
  end.uniq!

  Gitlab::Markdown::ReferenceFilter::LazyReference.load(references).uniq.sort_by(&:iid)
end
```

to

```ruby
def referenced_merge_requests
  Gitlab::ReferenceExtractor.lazily do
    [self, *notes].flat_map do |note|
      note.all_references.merge_requests
    end
  end.sort_by(&:iid)
end
```

See merge request !2027
parents 28a8d0b5 6560d053
...@@ -20,7 +20,7 @@ module GitlabMarkdownHelper ...@@ -20,7 +20,7 @@ module GitlabMarkdownHelper
end end
user = current_user if defined?(current_user) user = current_user if defined?(current_user)
gfm_body = Gitlab::Markdown.render(escaped_body, project: @project, current_user: user, pipeline: :single_line) gfm_body = Banzai.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body) fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
if fragment.children.size == 1 && fragment.children[0].name == 'a' if fragment.children.size == 1 && fragment.children[0].name == 'a'
...@@ -50,7 +50,7 @@ module GitlabMarkdownHelper ...@@ -50,7 +50,7 @@ module GitlabMarkdownHelper
context[:project] ||= @project context[:project] ||= @project
html = Gitlab::Markdown.render(text, context) html = Banzai.render(text, context)
context.merge!( context.merge!(
current_user: (current_user if defined?(current_user)), current_user: (current_user if defined?(current_user)),
...@@ -61,7 +61,7 @@ module GitlabMarkdownHelper ...@@ -61,7 +61,7 @@ module GitlabMarkdownHelper
ref: @ref ref: @ref
) )
Gitlab::Markdown.post_process(html, context) Banzai.post_process(html, context)
end end
def asciidoc(text) def asciidoc(text)
......
...@@ -121,6 +121,6 @@ module IssuesHelper ...@@ -121,6 +121,6 @@ module IssuesHelper
end end
end end
# Required for Gitlab::Markdown::IssueReferenceFilter # Required for Banzai::Filter::IssueReferenceFilter
module_function :url_for_issue module_function :url_for_issue
end end
...@@ -107,6 +107,6 @@ module LabelsHelper ...@@ -107,6 +107,6 @@ module LabelsHelper
options_from_collection_for_select(grouped_labels, 'name', 'title', params[:label_name]) options_from_collection_for_select(grouped_labels, 'name', 'title', params[:label_name])
end end
# Required for Gitlab::Markdown::LabelReferenceFilter # Required for Banzai::Filter::LabelReferenceFilter
module_function :render_colored_label, :text_color_for_bg, :escape_once module_function :render_colored_label, :text_color_for_bg, :escape_once
end end
...@@ -23,7 +23,7 @@ module Mentionable ...@@ -23,7 +23,7 @@ module Mentionable
included do included do
if self < Participable if self < Participable
participant ->(current_user) { mentioned_users(current_user, load_lazy_references: false) } participant ->(current_user) { mentioned_users(current_user) }
end end
end end
...@@ -43,8 +43,8 @@ module Mentionable ...@@ -43,8 +43,8 @@ module Mentionable
self self
end end
def all_references(current_user = self.author, text = nil, load_lazy_references: true) def all_references(current_user = self.author, text = nil)
ext = Gitlab::ReferenceExtractor.new(self.project, current_user, load_lazy_references: load_lazy_references) ext = Gitlab::ReferenceExtractor.new(self.project, current_user)
if text if text
ext.analyze(text) ext.analyze(text)
...@@ -59,13 +59,13 @@ module Mentionable ...@@ -59,13 +59,13 @@ module Mentionable
ext ext
end end
def mentioned_users(current_user = nil, load_lazy_references: true) def mentioned_users(current_user = nil)
all_references(current_user, load_lazy_references: load_lazy_references).users all_references(current_user).users
end end
# Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference. # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
def referenced_mentionables(current_user = self.author, text = nil, load_lazy_references: true) def referenced_mentionables(current_user = self.author, text = nil)
refs = all_references(current_user, text, load_lazy_references: load_lazy_references) refs = all_references(current_user, text)
refs = (refs.issues + refs.merge_requests + refs.commits) refs = (refs.issues + refs.merge_requests + refs.commits)
# We're using this method instead of Array diffing because that requires # We're using this method instead of Array diffing because that requires
......
...@@ -38,7 +38,9 @@ module Participable ...@@ -38,7 +38,9 @@ module Participable
# Be aware that this method makes a lot of sql queries. # Be aware that this method makes a lot of sql queries.
# Save result into variable if you are going to reuse it inside same request # Save result into variable if you are going to reuse it inside same request
def participants(current_user = self.author, load_lazy_references: true) def participants(current_user = self.author, load_lazy_references: true)
participants = self.class.participant_attrs.flat_map do |attr| participants =
Gitlab::ReferenceExtractor.lazily do
self.class.participant_attrs.flat_map do |attr|
value = value =
if attr.respond_to?(:call) if attr.respond_to?(:call)
instance_exec(current_user, &attr) instance_exec(current_user, &attr)
...@@ -48,10 +50,9 @@ module Participable ...@@ -48,10 +50,9 @@ module Participable
participants_for(value, current_user) participants_for(value, current_user)
end.compact.uniq end.compact.uniq
end
if load_lazy_references unless Gitlab::ReferenceExtractor.lazy?
participants = Gitlab::Markdown::ReferenceFilter::LazyReference.load(participants).uniq
participants.select! do |user| participants.select! do |user|
user.can?(:read_project, project) user.can?(:read_project, project)
end end
...@@ -64,12 +65,12 @@ module Participable ...@@ -64,12 +65,12 @@ module Participable
def participants_for(value, current_user = nil) def participants_for(value, current_user = nil)
case value case value
when User, Gitlab::Markdown::ReferenceFilter::LazyReference when User, Banzai::LazyReference
[value] [value]
when Enumerable, ActiveRecord::Relation when Enumerable, ActiveRecord::Relation
value.flat_map { |v| participants_for(v, current_user) } value.flat_map { |v| participants_for(v, current_user) }
when Participable when Participable
value.participants(current_user, load_lazy_references: false) value.participants(current_user)
end end
end end
end end
...@@ -84,11 +84,11 @@ class Issue < ActiveRecord::Base ...@@ -84,11 +84,11 @@ class Issue < ActiveRecord::Base
end end
def referenced_merge_requests def referenced_merge_requests
references = [self, *notes].flat_map do |note| Gitlab::ReferenceExtractor.lazily do
[self, *notes].flat_map do |note|
note.all_references(load_lazy_references: false).merge_requests note.all_references(load_lazy_references: false).merge_requests
end.uniq end
end.sort_by(&:iid)
Gitlab::Markdown::ReferenceFilter::LazyReference.load(references).uniq.sort_by(&:iid)
end end
# Reset issue events cache # Reset issue events cache
......
...@@ -373,11 +373,11 @@ class Note < ActiveRecord::Base ...@@ -373,11 +373,11 @@ class Note < ActiveRecord::Base
end end
def contains_emoji_only? def contains_emoji_only?
note =~ /\A#{Gitlab::Markdown::EmojiFilter.emoji_pattern}\s?\Z/ note =~ /\A#{Banzai::Filter::EmojiFilter.emoji_pattern}\s?\Z/
end end
def award_emoji_name def award_emoji_name
original_name = note.match(Gitlab::Markdown::EmojiFilter.emoji_pattern)[1] original_name = note.match(Banzai::Filter::EmojiFilter.emoji_pattern)[1]
AwardEmoji.normilize_emoji_name(original_name) AwardEmoji.normilize_emoji_name(original_name)
end end
end end
module Banzai
def self.render(text, context = {})
Renderer.render(text, context)
end
def self.render_result(text, context = {})
Renderer.render_result(text, context)
end
def self.post_process(html, context)
Renderer.post_process(html, context)
end
end
require 'banzai'
module Banzai
# Common methods for ReferenceFilters that support an optional cross-project
# reference.
module CrossProjectReference
# Given a cross-project reference string, get the Project record
#
# Defaults to value of `context[:project]` if:
# * No reference is given OR
# * Reference given doesn't exist
#
# ref - String reference.
#
# Returns a Project, or nil if the reference can't be found
def project_from_ref(ref)
return context[:project] unless ref
Project.find_with_namespace(ref)
end
end
end
require 'active_support/core_ext/string/output_safety'
require 'banzai'
module Banzai
module Filter
def self.[](name)
const_get("#{name.to_s.camelize}Filter")
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# Issues, Merge Requests, Snippets, Commits and Commit Ranges share # Issues, Merge Requests, Snippets, Commits and Commit Ranges share
# similar functionality in reference filtering. # similar functionality in reference filtering.
class AbstractReferenceFilter < ReferenceFilter class AbstractReferenceFilter < ReferenceFilter
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
require 'uri' require 'uri'
module Gitlab module Banzai
module Markdown module Filter
# HTML Filter for auto-linking URLs in HTML. # HTML Filter for auto-linking URLs in HTML.
# #
# Based on HTML::Pipeline::AutolinkFilter # Based on HTML::Pipeline::AutolinkFilter
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces commit range references with links. # HTML filter that replaces commit range references with links.
# #
# This filter supports cross-project references. # This filter supports cross-project references.
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces commit references with links. # HTML filter that replaces commit references with links.
# #
# This filter supports cross-project references. # This filter supports cross-project references.
......
require 'action_controller' require 'action_controller'
require 'gitlab/markdown' require 'banzai'
require 'gitlab_emoji' require 'gitlab_emoji'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces :emoji: with images. # HTML filter that replaces :emoji: with images.
# #
# Based on HTML::Pipeline::EmojiFilter # Based on HTML::Pipeline::EmojiFilter
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces external issue tracker references with links. # HTML filter that replaces external issue tracker references with links.
# References are ignored if the project doesn't use an external issue # References are ignored if the project doesn't use an external issue
# tracker. # tracker.
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# HTML Filter to add a `rel="nofollow"` attribute to external links # HTML Filter to add a `rel="nofollow"` attribute to external links
# #
class ExternalLinkFilter < HTML::Pipeline::Filter class ExternalLinkFilter < HTML::Pipeline::Filter
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces issue references with links. References to # HTML filter that replaces issue references with links. References to
# issues that do not exist are ignored. # issues that do not exist are ignored.
# #
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces label references with links. # HTML filter that replaces label references with links.
class LabelReferenceFilter < ReferenceFilter class LabelReferenceFilter < ReferenceFilter
# Public: Find label references in text # Public: Find label references in text
......
module Gitlab require 'banzai'
module Markdown require 'html/pipeline/filter'
module Banzai
module Filter
class MarkdownFilter < HTML::Pipeline::TextFilter class MarkdownFilter < HTML::Pipeline::TextFilter
def initialize(text, context = nil, result = nil) def initialize(text, context = nil, result = nil)
super text, context, result super text, context, result
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces merge request references with links. References # HTML filter that replaces merge request references with links. References
# to merge requests that do not exist are ignored. # to merge requests that do not exist are ignored.
# #
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that removes references to records that the current user does # HTML filter that removes references to records that the current user does
# not have permission to view. # not have permission to view.
# #
...@@ -27,7 +27,7 @@ module Gitlab ...@@ -27,7 +27,7 @@ module Gitlab
def user_can_reference?(node) def user_can_reference?(node)
if node.has_attribute?('data-reference-filter') if node.has_attribute?('data-reference-filter')
reference_type = node.attr('data-reference-filter') reference_type = node.attr('data-reference-filter')
reference_filter = Gitlab::Markdown.const_get(reference_type) reference_filter = Banzai::Filter.const_get(reference_type)
reference_filter.user_can_reference?(current_user, node, context) reference_filter.user_can_reference?(current_user, node, context)
else else
......
require 'active_support/core_ext/string/output_safety' require 'active_support/core_ext/string/output_safety'
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# Base class for GitLab Flavored Markdown reference filters. # Base class for GitLab Flavored Markdown reference filters.
# #
# References within <pre>, <code>, <a>, and <style> elements are ignored. # References within <pre>, <code>, <a>, and <style> elements are ignored.
...@@ -12,27 +12,6 @@ module Gitlab ...@@ -12,27 +12,6 @@ module Gitlab
# :project (required) - Current project, ignored if reference is cross-project. # :project (required) - Current project, ignored if reference is cross-project.
# :only_path - Generate path-only links. # :only_path - Generate path-only links.
class ReferenceFilter < HTML::Pipeline::Filter class ReferenceFilter < HTML::Pipeline::Filter
LazyReference = Struct.new(:klass, :ids) do
def self.load(refs)
lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
ids = refs.flat_map(&:ids)
klass.where(id: ids)
end
values + lazy_values
end
def load
self.klass.where(id: self.ids)
end
end
def self.[](name)
Markdown.const_get("#{name.to_s.camelize}ReferenceFilter")
end
def self.user_can_reference?(user, node, context) def self.user_can_reference?(user, node, context)
if node.has_attribute?('data-project') if node.has_attribute?('data-project')
project_id = node.attr('data-project').to_i project_id = node.attr('data-project').to_i
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that gathers all referenced records that the current user has # HTML filter that gathers all referenced records that the current user has
# permission to view. # permission to view.
# #
...@@ -20,7 +20,7 @@ module Gitlab ...@@ -20,7 +20,7 @@ module Gitlab
gather_references(node) gather_references(node)
end end
load_lazy_references unless context[:load_lazy_references] == false load_lazy_references unless ReferenceExtractor.lazy?
doc doc
end end
...@@ -31,7 +31,7 @@ module Gitlab ...@@ -31,7 +31,7 @@ module Gitlab
return unless node.has_attribute?('data-reference-filter') return unless node.has_attribute?('data-reference-filter')
reference_type = node.attr('data-reference-filter') reference_type = node.attr('data-reference-filter')
reference_filter = Gitlab::Markdown.const_get(reference_type) reference_filter = Banzai::Filter.const_get(reference_type)
return if context[:reference_filter] && reference_filter != context[:reference_filter] return if context[:reference_filter] && reference_filter != context[:reference_filter]
...@@ -47,11 +47,10 @@ module Gitlab ...@@ -47,11 +47,10 @@ module Gitlab
end end
end end
# Will load all references of one type using one query.
def load_lazy_references def load_lazy_references
refs = result[:references] refs = result[:references]
refs.each do |type, values| refs.each do |type, values|
refs[type] = ReferenceFilter::LazyReference.load(values) refs[type] = ReferenceExtractor.lazily(values)
end end
end end
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
require 'uri' require 'uri'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that "fixes" relative links to files in a repository. # HTML filter that "fixes" relative links to files in a repository.
# #
# Context options: # Context options:
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
require 'html/pipeline/sanitization_filter' require 'html/pipeline/sanitization_filter'
module Gitlab module Banzai
module Markdown module Filter
# Sanitize HTML # Sanitize HTML
# #
# Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist.
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces snippet references with links. References to # HTML filter that replaces snippet references with links. References to
# snippets that do not exist are ignored. # snippets that do not exist are ignored.
# #
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
require 'rouge/plugins/redcarpet' require 'rouge/plugins/redcarpet'
module Gitlab module Banzai
module Markdown module Filter
# HTML Filter to highlight fenced code blocks # HTML Filter to highlight fenced code blocks
# #
class SyntaxHighlightFilter < HTML::Pipeline::Filter class SyntaxHighlightFilter < HTML::Pipeline::Filter
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that adds an anchor child element to all Headers in a # HTML filter that adds an anchor child element to all Headers in a
# document, so that they can be linked to. # document, so that they can be linked to.
# #
......
require 'gitlab/markdown' require 'banzai'
require 'task_list/filter' require 'task_list/filter'
module Gitlab module Banzai
module Markdown module Filter
# Work around a bug in the default TaskList::Filter that adds a `task-list` # Work around a bug in the default TaskList::Filter that adds a `task-list`
# class to every list element, regardless of whether or not it contains a # class to every list element, regardless of whether or not it contains a
# task list. # task list.
......
require 'gitlab/markdown' require 'banzai'
require 'html/pipeline/filter' require 'html/pipeline/filter'
require 'uri' require 'uri'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that "fixes" relative upload links to files. # HTML filter that "fixes" relative upload links to files.
# Context options: # Context options:
# :project (required) - Current project # :project (required) - Current project
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Filter
# HTML filter that replaces user or group references with links. # HTML filter that replaces user or group references with links.
# #
# A special `@all` reference is also supported. # A special `@all` reference is also supported.
......
require 'banzai'
module Banzai
class LazyReference
def self.load(refs)
lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
ids = refs.flat_map(&:ids)
klass.where(id: ids)
end
values + lazy_values
end
attr_reader :klass, :ids
def initialize(klass, ids)
@klass = klass
@ids = Array.wrap(ids).map(&:to_i)
end
def load
self.klass.where(id: self.ids)
end
end
end
require 'banzai'
module Banzai
module Pipeline
def self.[](name)
name ||= :full
const_get("#{name.to_s.camelize}Pipeline")
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class AsciidocPipeline < Pipeline class AsciidocPipeline < BasePipeline
def self.filters def self.filters
[ [
Gitlab::Markdown::RelativeLinkFilter Filter::RelativeLinkFilter
] ]
end end
end end
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class AtomPipeline < FullPipeline class AtomPipeline < FullPipeline
def self.transform_context(context) def self.transform_context(context)
super(context).merge( super(context).merge(
......
require 'banzai'
require 'html/pipeline'
module Banzai
module Pipeline
class BasePipeline
def self.filters
[]
end
def self.transform_context(context)
context
end
def self.html_pipeline
@html_pipeline ||= HTML::Pipeline.new(filters)
end
class << self
%i(call to_document to_html).each do |meth|
define_method(meth) do |text, context|
context = transform_context(context)
html_pipeline.send(meth, text, context)
end
end
end
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
module CombinedPipeline module CombinedPipeline
def self.new(*pipelines) def self.new(*pipelines)
Class.new(Pipeline) do Class.new(BasePipeline) do
const_set :PIPELINES, pipelines const_set :PIPELINES, pipelines
def self.pipelines def self.pipelines
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class DescriptionPipeline < FullPipeline class DescriptionPipeline < FullPipeline
def self.transform_context(context) def self.transform_context(context)
super(context).merge( super(context).merge(
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class EmailPipeline < FullPipeline class EmailPipeline < FullPipeline
def self.transform_context(context) def self.transform_context(context)
super(context).merge( super(context).merge(
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
end end
......
require 'banzai'
module Banzai
module Pipeline
class GfmPipeline < BasePipeline
def self.filters
@filters ||= [
Filter::SyntaxHighlightFilter,
Filter::SanitizationFilter,
Filter::UploadLinkFilter,
Filter::EmojiFilter,
Filter::TableOfContentsFilter,
Filter::AutolinkFilter,
Filter::ExternalLinkFilter,
Filter::UserReferenceFilter,
Filter::IssueReferenceFilter,
Filter::ExternalIssueReferenceFilter,
Filter::MergeRequestReferenceFilter,
Filter::SnippetReferenceFilter,
Filter::CommitRangeReferenceFilter,
Filter::CommitReferenceFilter,
Filter::LabelReferenceFilter,
Filter::TaskListFilter
]
end
def self.transform_context(context)
context.merge(
only_path: true,
# EmojiFilter
asset_host: Gitlab::Application.config.asset_host,
asset_root: Gitlab.config.gitlab.base_url
)
end
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class NotePipeline < FullPipeline class NotePipeline < FullPipeline
def self.transform_context(context) def self.transform_context(context)
super(context).merge( super(context).merge(
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class PlainMarkdownPipeline < Pipeline class PlainMarkdownPipeline < BasePipeline
def self.filters def self.filters
[ [
Gitlab::Markdown::MarkdownFilter Filter::MarkdownFilter
] ]
end end
end end
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class PostProcessPipeline < Pipeline class PostProcessPipeline < BasePipeline
def self.filters def self.filters
[ [
Gitlab::Markdown::RelativeLinkFilter, Filter::RelativeLinkFilter,
Gitlab::Markdown::RedactorFilter Filter::RedactorFilter
] ]
end end
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class ReferenceExtractionPipeline < Pipeline class ReferenceExtractionPipeline < BasePipeline
def self.filters def self.filters
[ [
Gitlab::Markdown::ReferenceGathererFilter Filter::ReferenceGathererFilter
] ]
end end
end end
......
require 'gitlab/markdown' require 'banzai'
module Gitlab module Banzai
module Markdown module Pipeline
class SingleLinePipeline < GfmPipeline class SingleLinePipeline < GfmPipeline
end end
......
require 'banzai'
module Banzai
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor
class << self
LAZY_KEY = :banzai_reference_extractor_lazy
def lazy?
Thread.current[LAZY_KEY]
end
def lazily(values = nil, &block)
return (values || block.call).uniq if lazy?
begin
Thread.current[LAZY_KEY] = true
values ||= block.call
Banzai::LazyReference.load(values.uniq).uniq
ensure
Thread.current[LAZY_KEY] = false
end
end
end
def initialize
@texts = []
end
def analyze(text, context = {})
@texts << Renderer.render(text, context)
end
def references(type, context = {})
filter = Banzai::Filter["#{type}_reference"]
context.merge!(
pipeline: :reference_extraction,
# ReferenceGathererFilter
reference_filter: filter
)
self.class.lazily do
@texts.flat_map do |html|
text_context = context.dup
result = Renderer.render_result(html, text_context)
result[:references][type]
end.uniq
end
end
end
end
require 'html/pipeline' module Banzai
module Renderer
module Gitlab
# Custom parser for GitLab-flavored Markdown
#
# See the files in `lib/gitlab/markdown/` for specific processing information.
module Markdown
# Convert a Markdown String into an HTML-safe String of HTML # Convert a Markdown String into an HTML-safe String of HTML
# #
# Note that while the returned HTML will have been sanitized of dangerous # Note that while the returned HTML will have been sanitized of dangerous
...@@ -75,41 +70,7 @@ module Gitlab ...@@ -75,41 +70,7 @@ module Gitlab
def self.full_cache_key(cache_key, pipeline_name) def self.full_cache_key(cache_key, pipeline_name)
return unless cache_key return unless cache_key
["markdown", *cache_key, pipeline_name || :full] ["banzai", *cache_key, pipeline_name || :full]
end end
# Provide autoload paths for filters to prevent a circular dependency error
autoload :AutolinkFilter, 'gitlab/markdown/filter/autolink_filter'
autoload :CommitRangeReferenceFilter, 'gitlab/markdown/filter/commit_range_reference_filter'
autoload :CommitReferenceFilter, 'gitlab/markdown/filter/commit_reference_filter'
autoload :EmojiFilter, 'gitlab/markdown/filter/emoji_filter'
autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/filter/external_issue_reference_filter'
autoload :ExternalLinkFilter, 'gitlab/markdown/filter/external_link_filter'
autoload :IssueReferenceFilter, 'gitlab/markdown/filter/issue_reference_filter'
autoload :LabelReferenceFilter, 'gitlab/markdown/filter/label_reference_filter'
autoload :MarkdownFilter, 'gitlab/markdown/filter/markdown_filter'
autoload :MergeRequestReferenceFilter, 'gitlab/markdown/filter/merge_request_reference_filter'
autoload :RedactorFilter, 'gitlab/markdown/filter/redactor_filter'
autoload :ReferenceGathererFilter, 'gitlab/markdown/filter/reference_gatherer_filter'
autoload :RelativeLinkFilter, 'gitlab/markdown/filter/relative_link_filter'
autoload :SanitizationFilter, 'gitlab/markdown/filter/sanitization_filter'
autoload :SnippetReferenceFilter, 'gitlab/markdown/filter/snippet_reference_filter'
autoload :SyntaxHighlightFilter, 'gitlab/markdown/filter/syntax_highlight_filter'
autoload :TableOfContentsFilter, 'gitlab/markdown/filter/table_of_contents_filter'
autoload :TaskListFilter, 'gitlab/markdown/filter/task_list_filter'
autoload :UserReferenceFilter, 'gitlab/markdown/filter/user_reference_filter'
autoload :UploadLinkFilter, 'gitlab/markdown/filter/upload_link_filter'
autoload :AsciidocPipeline, 'gitlab/markdown/pipeline/asciidoc_pipeline'
autoload :AtomPipeline, 'gitlab/markdown/pipeline/atom_pipeline'
autoload :DescriptionPipeline, 'gitlab/markdown/pipeline/description_pipeline'
autoload :EmailPipeline, 'gitlab/markdown/pipeline/email_pipeline'
autoload :FullPipeline, 'gitlab/markdown/pipeline/full_pipeline'
autoload :GfmPipeline, 'gitlab/markdown/pipeline/gfm_pipeline'
autoload :NotePipeline, 'gitlab/markdown/pipeline/note_pipeline'
autoload :PlainMarkdownPipeline, 'gitlab/markdown/pipeline/plain_markdown_pipeline'
autoload :PostProcessPipeline, 'gitlab/markdown/pipeline/post_process_pipeline'
autoload :ReferenceExtractionPipeline, 'gitlab/markdown/pipeline/reference_extraction_pipeline'
autoload :SingleLinePipeline, 'gitlab/markdown/pipeline/single_line_pipeline'
end end
end end
...@@ -32,7 +32,7 @@ module Gitlab ...@@ -32,7 +32,7 @@ module Gitlab
html = ::Asciidoctor.convert(input, asciidoc_opts) html = ::Asciidoctor.convert(input, asciidoc_opts)
if context[:project] if context[:project]
html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc)) html = Banzai.render(html, context.merge(pipeline: :asciidoc))
end end
html.html_safe html.html_safe
......
require 'gitlab/markdown'
module Gitlab
module Markdown
# Common methods for ReferenceFilters that support an optional cross-project
# reference.
module CrossProjectReference
# Given a cross-project reference string, get the Project record
#
# Defaults to value of `context[:project]` if:
# * No reference is given OR
# * Reference given doesn't exist
#
# ref - String reference.
#
# Returns a Project, or nil if the reference can't be found
def project_from_ref(ref)
return context[:project] unless ref
Project.find_with_namespace(ref)
end
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Gitlab
module Markdown module Markdown
class Pipeline class Pipeline
def self.[](name) def self.[](name)
name ||= :full name ||= :full
Markdown.const_get("#{name.to_s.camelize}Pipeline") const_get("#{name.to_s.camelize}Pipeline")
end end
def self.filters def self.filters
......
require 'gitlab/markdown'
module Gitlab
module Markdown
class GfmPipeline < Pipeline
def self.filters
@filters ||= [
Gitlab::Markdown::SyntaxHighlightFilter,
Gitlab::Markdown::SanitizationFilter,
Gitlab::Markdown::UploadLinkFilter,
Gitlab::Markdown::EmojiFilter,
Gitlab::Markdown::TableOfContentsFilter,
Gitlab::Markdown::AutolinkFilter,
Gitlab::Markdown::ExternalLinkFilter,
Gitlab::Markdown::UserReferenceFilter,
Gitlab::Markdown::IssueReferenceFilter,
Gitlab::Markdown::ExternalIssueReferenceFilter,
Gitlab::Markdown::MergeRequestReferenceFilter,
Gitlab::Markdown::SnippetReferenceFilter,
Gitlab::Markdown::CommitRangeReferenceFilter,
Gitlab::Markdown::CommitReferenceFilter,
Gitlab::Markdown::LabelReferenceFilter,
Gitlab::Markdown::TaskListFilter
]
end
def self.transform_context(context)
context.merge(
only_path: true,
# EmojiFilter
asset_host: Gitlab::Application.config.asset_host,
asset_root: Gitlab.config.gitlab.base_url
)
end
end
end
end
require 'gitlab/markdown' require 'banzai'
module Gitlab module Gitlab
# Extract possible GFM references from an arbitrary String for further processing. # Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor class ReferenceExtractor < Banzai::ReferenceExtractor
attr_accessor :project, :current_user, :load_lazy_references attr_accessor :project, :current_user
def initialize(project, current_user = nil, load_lazy_references: true) def initialize(project, current_user = nil)
@project = project @project = project
@current_user = current_user @current_user = current_user
@load_lazy_references = load_lazy_references
@texts = []
@references = {} @references = {}
super()
end end
def analyze(text, options = {}) def analyze(text, context = {})
@texts << Gitlab::Markdown.render(text, options.merge(project: project)) super(text, context.merge(project: project))
end end
%i(user label issue merge_request snippet commit commit_range).each do |type| %i(user label issue merge_request snippet commit commit_range).each do |type|
define_method("#{type}s") do define_method("#{type}s") do
@references[type] ||= pipeline_result(type) @references[type] ||= references(type, project: project, current_user: current_user)
end
end end
private
# Instantiate and call HTML::Pipeline with a single reference filter type,
# returning the result
#
# filter_type - Symbol reference type (e.g., :commit, :issue, etc.)
#
# Returns the results Array for the requested filter type
def pipeline_result(filter_type)
filter = Gitlab::Markdown::ReferenceFilter[filter_type]
context = {
pipeline: :reference_extraction,
project: project,
current_user: current_user,
# ReferenceGathererFilter
load_lazy_references: false,
reference_filter: filter
}
values = @texts.flat_map do |html|
text_context = context.dup
result = Gitlab::Markdown.render_result(html, text_context)
result[:references][filter_type]
end.uniq
if @load_lazy_references
values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq
end
values
end end
end end
end end
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::ReferenceFilter, benchmark: true do describe Banzai::Filter::ReferenceFilter, benchmark: true do
let(:input) do let(:input) do
html = <<-EOF html = <<-EOF
<p>Hello @alice and @bob, how are you doing today?</p> <p>Hello @alice and @bob, how are you doing today?</p>
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::CrossProjectReference, lib: true do describe Banzai::CrossProjectReference, lib: true do
include described_class include described_class
describe '#project_from_ref' do describe '#project_from_ref' do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::AutolinkFilter, lib: true do describe Banzai::Filter::AutolinkFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:link) { 'http://about.gitlab.com/' } let(:link) { 'http://about.gitlab.com/' }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::CommitRangeReferenceFilter, lib: true do describe Banzai::Filter::CommitRangeReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:project, :public) } let(:project) { create(:project, :public) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::CommitReferenceFilter, lib: true do describe Banzai::Filter::CommitReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:project, :public) } let(:project) { create(:project, :public) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::EmojiFilter, lib: true do describe Banzai::Filter::EmojiFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
before do before do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::ExternalIssueReferenceFilter, lib: true do describe Banzai::Filter::ExternalIssueReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
def helper def helper
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::ExternalLinkFilter, lib: true do describe Banzai::Filter::ExternalLinkFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
it 'ignores elements without an href attribute' do it 'ignores elements without an href attribute' do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::IssueReferenceFilter, lib: true do describe Banzai::Filter::IssueReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
def helper def helper
......
require 'spec_helper' require 'spec_helper'
require 'html/pipeline' require 'html/pipeline'
describe Gitlab::Markdown::LabelReferenceFilter, lib: true do describe Banzai::Filter::LabelReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:empty_project, :public) } let(:project) { create(:empty_project, :public) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::MergeRequestReferenceFilter, lib: true do describe Banzai::Filter::MergeRequestReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:project, :public) } let(:project) { create(:project, :public) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::RedactorFilter, lib: true do describe Banzai::Filter::RedactorFilter, lib: true do
include ActionView::Helpers::UrlHelper include ActionView::Helpers::UrlHelper
include FilterSpecHelper include FilterSpecHelper
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::ReferenceGathererFilter, lib: true do describe Banzai::Filter::ReferenceGathererFilter, lib: true do
include ActionView::Helpers::UrlHelper include ActionView::Helpers::UrlHelper
include FilterSpecHelper include FilterSpecHelper
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::RelativeLinkFilter, lib: true do describe Banzai::Filter::RelativeLinkFilter, lib: true do
def filter(doc, contexts = {}) def filter(doc, contexts = {})
contexts.reverse_merge!({ contexts.reverse_merge!({
commit: project.commit, commit: project.commit,
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::SanitizationFilter, lib: true do describe Banzai::Filter::SanitizationFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
describe 'default whitelist' do describe 'default whitelist' do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::SnippetReferenceFilter, lib: true do describe Banzai::Filter::SnippetReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:empty_project, :public) } let(:project) { create(:empty_project, :public) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::SyntaxHighlightFilter, lib: true do describe Banzai::Filter::SyntaxHighlightFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
it 'highlights valid code blocks' do it 'highlights valid code blocks' do
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::TableOfContentsFilter, lib: true do describe Banzai::Filter::TableOfContentsFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
def header(level, text) def header(level, text)
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::TaskListFilter, lib: true do describe Banzai::Filter::TaskListFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
it 'does not apply `task-list` class to non-task lists' do it 'does not apply `task-list` class to non-task lists' do
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::UploadLinkFilter, lib: true do describe Banzai::Filter::UploadLinkFilter, lib: true do
def filter(doc, contexts = {}) def filter(doc, contexts = {})
contexts.reverse_merge!({ contexts.reverse_merge!({
project: project project: project
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Markdown::UserReferenceFilter, lib: true do describe Banzai::Filter::UserReferenceFilter, lib: true do
include FilterSpecHelper include FilterSpecHelper
let(:project) { create(:empty_project, :public) } let(:project) { create(:empty_project, :public) }
......
...@@ -50,7 +50,7 @@ module Gitlab ...@@ -50,7 +50,7 @@ module Gitlab
filtered_html = '<b>ASCII</b>' filtered_html = '<b>ASCII</b>'
allow(Asciidoctor).to receive(:convert).and_return(html) allow(Asciidoctor).to receive(:convert).and_return(html)
expect(Gitlab::Markdown).to receive(:render) expect(Banzai).to receive(:render)
.with(html, context.merge(pipeline: :asciidoc)) .with(html, context.merge(pipeline: :asciidoc))
.and_return(filtered_html) .and_return(filtered_html)
......
# Helper methods for Gitlab::Markdown filter specs # Helper methods for Banzai filter specs
# #
# Must be included into specs manually # Must be included into specs manually
module FilterSpecHelper module FilterSpecHelper
...@@ -10,49 +10,49 @@ module FilterSpecHelper ...@@ -10,49 +10,49 @@ module FilterSpecHelper
# if none is provided. # if none is provided.
# #
# html - HTML String to pass to the filter's `call` method. # html - HTML String to pass to the filter's `call` method.
# contexts - Hash context for the filter. (default: {project: project}) # context - Hash context for the filter. (default: {project: project})
# #
# Returns a Nokogiri::XML::DocumentFragment # Returns a Nokogiri::XML::DocumentFragment
def filter(html, contexts = {}) def filter(html, context = {})
if defined?(project) if defined?(project)
contexts.reverse_merge!(project: project) context.reverse_merge!(project: project)
end end
described_class.call(html, contexts) described_class.call(html, context)
end end
# Run text through HTML::Pipeline with the current filter and return the # Run text through HTML::Pipeline with the current filter and return the
# result Hash # result Hash
# #
# body - String text to run through the pipeline # body - String text to run through the pipeline
# contexts - Hash context for the filter. (default: {project: project}) # context - Hash context for the filter. (default: {project: project})
# #
# Returns the Hash # Returns the Hash
def pipeline_result(body, contexts = {}) def pipeline_result(body, context = {})
contexts.reverse_merge!(project: project) if defined?(project) context.reverse_merge!(project: project) if defined?(project)
pipeline = HTML::Pipeline.new([described_class], contexts) pipeline = HTML::Pipeline.new([described_class], context)
pipeline.call(body) pipeline.call(body)
end end
def reference_pipeline(contexts = {}) def reference_pipeline(context = {})
contexts.reverse_merge!(project: project) if defined?(project) context.reverse_merge!(project: project) if defined?(project)
filters = [ filters = [
Gitlab::Markdown::AutolinkFilter, Banzai::Filter::AutolinkFilter,
described_class, described_class,
Gitlab::Markdown::ReferenceGathererFilter Banzai::Filter::ReferenceGathererFilter
] ]
HTML::Pipeline.new(filters, contexts) HTML::Pipeline.new(filters, context)
end end
def reference_pipeline_result(body, contexts = {}) def reference_pipeline_result(body, context = {})
reference_pipeline(contexts).call(body) reference_pipeline(context).call(body)
end end
def reference_filter(html, contexts = {}) def reference_filter(html, context = {})
reference_pipeline(contexts).to_document(html) reference_pipeline(context).to_document(html)
end end
# Modify a String reference to make it invalid # Modify a String reference to make it invalid
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment