Skip to content

G
gitlab-ce
Commit fbd75c35 authored by Robert Speicher's avatar Robert Speicher
Browse files
Options

Merge branch 'feature.rouge-20' into 'master' 

Upgrade to Rouge 2.0

## What does this MR do?

This MR is a refactor of the HTMLGitlab formatter. The original was copy-pasted from the legacy HTML formatter and adapted to suit our needs. In particular, this MR:

* Strips dead code and unused options
* Factors out the "wrapping" into the places that care about it - Bonzai and the view helpers
* Uses the Rouge 2.0 `token_lines` method to split tokens into lines, removing the custom code we used to do the same
* Uses the now-public `span` method instead of re-implementing it ourselves
* Removes options to not split into lines - the places where this feature wasn't being used are now using the plain `Formatters::HTML` instead of this class

See merge request !4691
parents 27e4a952 c4ea3947
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 236 deletions
Gemfile
View file @ fbd75c35
......@@ -61,7 +61,7 @@ gem 'gitlab_omniauth-ldap', '~> 1.2.1', require: 'omniauth-ldap'
# Git Wiki
# Required manually in config/initializers/gollum.rb to control load order
gem 'gollum-lib', '~> 4.1.0', require: false
gem 'gollum-lib', '~> 4.2', require: false
gem 'gollum-rugged_adapter', '~> 0.4.2', require: false
# Language detection
......@@ -105,7 +105,7 @@ gem 'seed-fu', '~> 2.3.5'
# Markdown and HTML processing
gem 'html-pipeline', '~> 1.11.0'
gem 'task_list', '~> 1.0.2', require: 'task_list/railtie'
gem 'github-markup', '~> 1.3.1'
gem 'github-markup', '~> 1.4'
gem 'redcarpet', '~> 3.3.3'
gem 'RedCloth', '~> 4.3.2'
gem 'rdoc', '~>3.6'
......@@ -113,7 +113,7 @@ gem 'org-ruby', '~> 0.9.12'
gem 'creole', '~> 0.5.0'
gem 'wikicloth', '0.8.1'
gem 'asciidoctor', '~> 1.5.2'
gem 'rouge', '~> 1.11'
gem 'rouge', '~> 2.0'
# See https://groups.google.com/forum/#!topic/ruby-security-ann/aSbgDiwb24s
# and https://groups.google.com/forum/#!topic/ruby-security-ann/Dy7YiKb_pMM
......
Gemfile.lock
View file @ fbd75c35
......@@ -264,7 +264,7 @@ GEM
escape_utils (~> 1.1.0)
mime-types (>= 1.19)
rugged (>= 0.23.0b)
github-markup (1.3.3)
github-markup (1.4.0)
gitlab-flowdock-git-hook (1.0.1)
flowdock (~> 0.7)
gitlab-grit (>= 2.4.1)
......@@ -287,13 +287,13 @@ GEM
rubyntlm (~> 0.3)
globalid (0.3.6)
activesupport (>= 4.1.0)
gollum-grit_adapter (1.0.0)
gollum-grit_adapter (1.0.1)
gitlab-grit (~> 2.7, >= 2.7.1)
gollum-lib (4.1.0)
github-markup (~> 1.3.3)
gollum-lib (4.2.1)
github-markup (~> 1.4.0)
gollum-grit_adapter (~> 1.0)
nokogiri (~> 1.6.4)
rouge (~> 1.9)
rouge (~> 2.0)
sanitize (~> 2.1.0)
stringex (~> 2.5.1)
gollum-rugged_adapter (0.4.2)
......@@ -578,7 +578,7 @@ GEM
railties (>= 4.2.0, < 5.1)
rinku (2.0.0)
rotp (2.1.2)
rouge (1.11.0)
rouge (2.0.3)
rqrcode (0.7.0)
chunky_png
rqrcode-rails3 (0.1.7)
......@@ -859,12 +859,12 @@ DEPENDENCIES
gemnasium-gitlab-service (~> 0.2)
gemojione (~> 2.6)
github-linguist (~> 4.7.0)
github-markup (~> 1.3.1)
github-markup (~> 1.4)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab_git (~> 10.2)
gitlab_meta (= 7.0)
gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.1.0)
gollum-lib (~> 4.2)
gollum-rugged_adapter (~> 0.4.2)
gon (~> 6.0.1)
grape (~> 0.13.0)
......@@ -933,7 +933,7 @@ DEPENDENCIES
request_store (~> 1.3.0)
rerun (~> 0.11.0)
responders (~> 2.0)
rouge (~> 1.11)
rouge (~> 2.0)
rqrcode-rails3 (~> 0.1.7)
rspec-rails (~> 3.5.0)
rspec-retry (~> 0.4.5)
......
app/helpers/blob_helper.rb
View file @ fbd75c35
module BlobHelper
def highlighter(blob_name, blob_content, repository: nil, nowrap: false)
Gitlab::Highlight.new(blob_name, blob_content, nowrap: nowrap, repository: repository)
end
def highlight(blob_name, blob_content, repository: nil, nowrap: false, plain: false)
Gitlab::Highlight.highlight(blob_name, blob_content, nowrap: nowrap, plain: plain, repository: repository)
def highlight(blob_name, blob_content, repository: nil, plain: false)
highlighted = Gitlab::Highlight.highlight(blob_name, blob_content, plain: plain, repository: repository)
raw %(<pre class="code highlight"><code>#{highlighted}</code></pre>)
end
def no_highlight_files
......
lib/banzai/filter/syntax_highlight_filter.rb
View file @ fbd75c35
......@@ -19,8 +19,15 @@ module Banzai
language = node.attr('class')
code = node.text
lexer = Rouge::Lexer.find_fancy(language)
formatter = Rouge::Formatters::HTML.new
css_classes = "code highlight js-syntax-highlight #{lexer.tag}"
begin
highlighted = block_code(code, language)
highlighted = ''
highlighted << %(<pre class="#{css_classes}"><code>)
highlighted << formatter.format(lexer.lex(code))
highlighted << %(</code></pre>)
rescue
# Gracefully handle syntax highlighter bugs/errors to ensure
# users can still access an issue/comment/etc.
......@@ -40,8 +47,7 @@ module Banzai
# Override Rouge::Plugins::Redcarpet#rouge_formatter
def rouge_formatter(lexer)
Rouge::Formatters::HTMLGitlab.new(
cssclass: "code highlight js-syntax-highlight #{lexer.tag}")
Rouge::Formatters::HTML.new
end
end
end
......
lib/gitlab/highlight.rb
View file @ fbd75c35
module Gitlab
class Highlight
def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false)
new(blob_name, blob_content, nowrap: nowrap, repository: repository).
def self.highlight(blob_name, blob_content, repository: nil, plain: false)
new(blob_name, blob_content, repository: repository).
highlight(blob_content, continue: false, plain: plain)
end
......@@ -13,30 +13,34 @@ module Gitlab
highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
end
attr_reader :lexer
def initialize(blob_name, blob_content, repository: nil, nowrap: true)
def initialize(blob_name, blob_content, repository: nil)
@formatter = Rouge::Formatters::HTMLGitlab.new
@repository = repository
@blob_name = blob_name
@blob_content = blob_content
@repository = repository
@formatter = rouge_formatter(nowrap: nowrap)
@lexer = custom_language || begin
Rouge::Lexer.guess(filename: blob_name, source: blob_content).new
rescue Rouge::Lexer::AmbiguousGuess => e
e.alternatives.sort_by(&:tag).first
end
end
def highlight(text, continue: true, plain: false)
if plain
@formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
hl_lexer = Rouge::Lexers::PlainText
continue = false
else
@formatter.format(@lexer.lex(text, continue: continue)).html_safe
hl_lexer = self.lexer
end
@formatter.format(hl_lexer.lex(text, continue: continue)).html_safe
rescue
@formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
end
def lexer
@lexer ||= custom_language || begin
Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new
rescue Rouge::Guesser::Ambiguous => e
e.alternatives.sort_by(&:tag).first
end
end
private
def custom_language
......@@ -46,16 +50,5 @@ module Gitlab
Rouge::Lexer.find_fancy(language_name)
end
def rouge_formatter(options = {})
options = options.reverse_merge(
nowrap: true,
cssclass: 'code highlight',
lineanchors: true,
lineanchorsid: 'LC'
)
Rouge::Formatters::HTMLGitlab.new(options)
end
end
end
lib/rouge/formatters/html_gitlab.rb
View file @ fbd75c35
require 'cgi'
module Rouge
module Formatters
class HTMLGitlab < Rouge::Formatter
class HTMLGitlab < Rouge::Formatters::HTML
tag 'html_gitlab'
# Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance.
#
# [+nowrap+] If set to True, don't wrap the output at all, not
# even inside a <tt><pre></tt> tag (default: false).
# [+cssclass+] CSS class for the wrapping <tt><div></tt> tag
# (default: 'highlight').
# [+linenos+] If set to 'table', output line numbers as a table
# with two cells, one containing the line numbers,
# the other the whole code. This is copy paste friendly,
# but may cause alignment problems with some browsers
# or fonts. If set to 'inline', the line numbers will
# be integrated in the <tt><pre></tt> tag that contains
# the code (default: nil).
# [+linenostart+] The line number for the first line (default: 1).
# [+lineanchors+] If set to true the formatter will wrap each output
# line in an anchor tag with a name of L-linenumber.
# This allows easy linking to certain lines
# (default: false).
# [+lineanchorsid+] If lineanchors is true the name of the anchors can
# be changed with lineanchorsid to e.g. foo-linenumber
# (default: 'L').
# [+anchorlinenos+] If set to true, will wrap line numbers in <tt><a></tt>
# tags. Used in combination with linenos and lineanchors
# (default: false).
# [+inline_theme+] Inline CSS styles for the <pre> tag (default: false).
def initialize(
nowrap: false,
cssclass: 'highlight',
linenos: nil,
linenostart: 1,
lineanchors: false,
lineanchorsid: 'L',
anchorlinenos: false,
inline_theme: nil
)
@nowrap = nowrap
@cssclass = cssclass
@linenos = linenos
def initialize(linenostart: 1)
@linenostart = linenostart
@lineanchors = lineanchors
@lineanchorsid = lineanchorsid
@anchorlinenos = anchorlinenos
@inline_theme = Theme.find(inline_theme).new if inline_theme.is_a?(String)
end
def render(tokens)
case @linenos
when 'table'
render_tableized(tokens)
when 'inline'
render_untableized(tokens)
else
render_untableized(tokens)
end
end
alias_method :format, :render
private
def render_untableized(tokens)
data = process_tokens(tokens)
html = ''
html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap
html << wrap_lines(data[:code])
html << "</code></pre>\n" unless @nowrap
html
@line_number = linenostart
end
def render_tableized(tokens)
data = process_tokens(tokens)
html = ''
html << "<div class=\"#{@cssclass}\">" unless @nowrap
html << '<table><tbody>'
html << "<td class=\"linenos\"><pre>"
html << wrap_linenos(data[:numbers])
html << '</pre></td>'
html << "<td class=\"lines\"><pre><code>"
html << wrap_lines(data[:code])
html << '</code></pre></td>'
html << '</tbody></table>'
html << '</div>' unless @nowrap
html
end
def process_tokens(tokens)
rendered = []
current_line = ''
tokens.each do |tok, val|
# In the case of multi-line values (e.g. comments), we need to apply
# styling to each line since span elements are inline.
val.lines.each do |line|
stripped = line.chomp
current_line << span(tok, stripped)
if line.end_with?("\n")
rendered << current_line
current_line = ''
end
end
end
# Add leftover text
rendered << current_line if current_line.present?
num_lines = rendered.size
numbers = (@linenostart..num_lines + @linenostart - 1).to_a
{ numbers: numbers, code: rendered }
end
def wrap_linenos(numbers)
if @anchorlinenos
numbers.map! do |number|
"<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>"
end
end
numbers.join("\n")
end
def wrap_lines(lines)
if @lineanchors
lines = lines.each_with_index.map do |line, index|
number = index + @linenostart
if @linenos == 'inline'
"<a name=\"L#{number}\"></a>" \
"<span class=\"linenos\">#{number}</span>" \
"<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
'</span>'
else
"<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
'</span>'
end
end
elsif @linenos == 'inline'
lines = lines.each_with_index.map do |line, index|
number = index + @linenostart
"<span class=\"linenos\">#{number}</span>#{line}"
end
end
lines.join("\n")
end
def stream(tokens, &b)
is_first = true
token_lines(tokens) do |line|
yield "\n" unless is_first
is_first = false
def span(tok, val)
# http://stackoverflow.com/a/1600584/2587286
val = CGI.escapeHTML(val)
yield %(<span id="LC#{@line_number}" class="line">)
line.each { |token, value| yield span(token, value) }
yield %(</span>)
if tok.shortname.empty?
val
else
if @inline_theme
rules = @inline_theme.style_for(tok).rendered_rules
"<span style=\"#{rules.to_a.join(';')}\"#{val}</span>"
else
"<span class=\"#{tok.shortname}\">#{val}</span>"
end
@line_number += 1
end
end
end
......
spec/fixtures/parallel_diff_result.yml
View file @ fbd75c35
......@@ -121,7 +121,7 @@
:type: old
:number: 9
:text: |
-<span id="LC9" class="line"> <span class="k">raise</span> <span class="s2">&quot;System commands must be given as an array of strings&quot;</span></span>
-<span id="LC9" class="line"> <span class="k">raise</span> <span class="s2">"System commands must be given as an array of strings"</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_9_9
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......@@ -136,7 +136,7 @@
:type: new
:number: 9
:text: |
+<span id="LC9" class="line"> <span class="k">raise</span> <span class="no"><span class='idiff left'>RuntimeError</span></span><span class="p"><span class='idiff'>,</span></span><span class='idiff right'> </span><span class="s2">&quot;System commands must be given as an array of strings&quot;</span></span>
+<span id="LC9" class="line"> <span class="k">raise</span> <span class="no"><span class='idiff left'>RuntimeError</span></span><span class="p"><span class='idiff'>,</span></span><span class='idiff right'> </span><span class="s2">"System commands must be given as an array of strings"</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_10_9
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......@@ -241,7 +241,7 @@
:type: old
:number: 13
:text: |
-<span id="LC13" class="line"> <span class="n">vars</span> <span class="o">=</span> <span class="p">{</span> <span class="s2">&quot;PWD&quot;</span> <span class="o">=&gt;</span> <span class="n">path</span> <span class="p">}</span></span>
-<span id="LC13" class="line"> <span class="n">vars</span> <span class="o">=</span> <span class="p">{</span> <span class="s2">"PWD"</span> <span class="o">=&gt;</span> <span class="n">path</span> <span class="p">}</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_13_13
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......@@ -315,7 +315,7 @@
:type: new
:number: 15
:text: |
+<span id="LC15" class="line"> <span class="s2">&quot;PWD&quot;</span> <span class="o">=&gt;</span> <span class="n">path</span></span>
+<span id="LC15" class="line"> <span class="s2">"PWD"</span> <span class="o">=&gt;</span> <span class="n">path</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_15_15
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......@@ -623,7 +623,7 @@
:type:
:number: 20
:text: |2
<span id="LC26" class="line"> <span class="vi">@cmd_output</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span></span>
<span id="LC26" class="line"> <span class="vi">@cmd_output</span> <span class="o">=</span> <span class="s2">""</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_20_26
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......@@ -638,7 +638,7 @@
:type:
:number: 26
:text: |2
<span id="LC26" class="line"> <span class="vi">@cmd_output</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span></span>
<span id="LC26" class="line"> <span class="vi">@cmd_output</span> <span class="o">=</span> <span class="s2">""</span></span>
:line_code: 2f6fcd96b88b36ce98c38da085c795a27d92a3dd_20_26
:position: !ruby/object:Gitlab::Diff::Position
attributes:
......
spec/helpers/blob_helper_spec.rb
View file @ fbd75c35
......@@ -16,19 +16,19 @@ describe BlobHelper do
describe '#highlight' do
it 'should return plaintext for unknown lexer context' do
result = helper.highlight(blob_name, no_context_content, nowrap: true)
expect(result).to eq('<span id="LC1" class="line">:type &quot;assem&quot;))</span>')
result = helper.highlight(blob_name, no_context_content)
expect(result).to eq(%[<pre class="code highlight"><code><span id="LC1" class="line">:type "assem"))</span></code></pre>])
end
it 'should highlight single block' do
expected = %Q[<span id="LC1" class="line"><span class="p">(</span><span class="nb">make-pathname</span> <span class="ss">:defaults</span> <span class="nv">name</span></span>
<span id="LC2" class="line"><span class="ss">:type</span> <span class="s">&quot;assem&quot;</span><span class="p">))</span></span>]
expected = %Q[<pre class="code highlight"><code><span id="LC1" class="line"><span class="p">(</span><span class="nb">make-pathname</span> <span class="ss">:defaults</span> <span class="nv">name</span></span>
<span id="LC2" class="line"><span class="ss">:type</span> <span class="s">"assem"</span><span class="p">))</span></span></code></pre>]
expect(helper.highlight(blob_name, blob_content, nowrap: true)).to eq(expected)
expect(helper.highlight(blob_name, blob_content)).to eq(expected)
end
it 'should highlight multi-line comments' do
result = helper.highlight(blob_name, multiline_content, nowrap: true)
result = helper.highlight(blob_name, multiline_content)
html = Nokogiri::HTML(result)
lines = html.search('.s')
expect(lines.count).to eq(3)
......@@ -41,33 +41,19 @@ describe BlobHelper do
let(:blob_name) { 'test.diff' }
let(:blob_content) { "+aaa\n+bbb\n- ccc\n ddd\n"}
let(:expected) do
%q(<span id="LC1" class="line"><span class="gi">+aaa</span></span>
%q(<pre class="code highlight"><code><span id="LC1" class="line"><span class="gi">+aaa</span></span>
<span id="LC2" class="line"><span class="gi">+bbb</span></span>
<span id="LC3" class="line"><span class="gd">- ccc</span></span>
<span id="LC4" class="line"> ddd</span>)
<span id="LC4" class="line"> ddd</span></code></pre>)
end
it 'should highlight each line properly' do
result = helper.highlight(blob_name, blob_content, nowrap: true)
result = helper.highlight(blob_name, blob_content)
expect(result).to eq(expected)
end
end
end
describe "#highlighter" do
it 'should highlight continued blocks' do
# Both lines have LC1 as ID since formatter doesn't support continue at the moment
expected = [
'<span id="LC1" class="line"><span class="p">(</span><span class="nb">make-pathname</span> <span class="ss">:defaults</span> <span class="nv">name</span></span>',
'<span id="LC1" class="line"><span class="ss">:type</span> <span class="s">&quot;assem&quot;</span><span class="p">))</span></span>'
]
highlighter = helper.highlighter(blob_name, blob_content, nowrap: true)
result = split_content.map{ |content| highlighter.highlight(content) }
expect(result).to eq(expected)
end
end
describe "#sanitize_svg" do
let(:input_svg_path) { File.join(Rails.root, 'spec', 'fixtures', 'unsanitized.svg') }
let(:data) { open(input_svg_path).read }
......
spec/helpers/events_helper_spec.rb
View file @ fbd75c35
......@@ -57,7 +57,7 @@ describe EventsHelper do
expected = '<pre class="code highlight js-syntax-highlight ruby">' \
"<code><span class=\"k\">def</span> <span class=\"nf\">test</span>\n" \
" <span class=\"s1\">\'hello world\'</span>\n" \
"<span class=\"k\">end</span>" \
"<span class=\"k\">end</span>\n" \
'</code></pre>'
expect(helper.event_note(input)).to eq(expected)
end
......
spec/lib/banzai/filter/syntax_highlight_filter_spec.rb
View file @ fbd75c35
......@@ -5,11 +5,11 @@ describe Banzai::Filter::SyntaxHighlightFilter, lib: true do
it 'highlights valid code blocks' do
result = filter('<pre><code>def fun end</code>')
expect(result.to_html).to eq("<pre class=\"code highlight js-syntax-highlight plaintext\"><code>def fun end</code></pre>\n")
expect(result.to_html).to eq("<pre class=\"code highlight js-syntax-highlight plaintext\"><code>def fun end</code></pre>")
end
it 'passes through invalid code blocks' do
allow_any_instance_of(described_class).to receive(:block_code).and_raise(StandardError)
allow_any_instance_of(Rouge::Formatter).to receive(:format).and_raise(StandardError)
result = filter('<pre><code>This is a test</code></pre>')
expect(result.to_html).to eq('<pre>This is a test</pre>')
......
spec/lib/gitlab/diff/highlight_spec.rb
View file @ fbd75c35
......@@ -28,13 +28,13 @@ describe Gitlab::Diff::Highlight, lib: true do
end
it 'highlights and marks removed lines' do
code = %Q{-<span id="LC9" class="line"> <span class="k">raise</span> <span class="s2">&quot;System commands must be given as an array of strings&quot;</span></span>\n}
code = %Q{-<span id="LC9" class="line"> <span class="k">raise</span> <span class="s2">"System commands must be given as an array of strings"</span></span>\n}
expect(subject[4].text).to eq(code)
end
it 'highlights and marks added lines' do
code = %Q{+<span id="LC9" class="line"> <span class="k">raise</span> <span class="no"><span class='idiff left'>RuntimeError</span></span><span class="p"><span class='idiff'>,</span></span><span class='idiff right'> </span><span class="s2">&quot;System commands must be given as an array of strings&quot;</span></span>\n}
code = %Q{+<span id="LC9" class="line"> <span class="k">raise</span> <span class="no"><span class='idiff left'>RuntimeError</span></span><span class="p"><span class='idiff'>,</span></span><span class='idiff right'> </span><span class="s2">"System commands must be given as an array of strings"</span></span>\n}
expect(subject[5].text).to eq(code)
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7