Commit 73868605 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'ee-add-language-param-to-highlight' into 'master'

(EE-Port) Add language param to highlight

See merge request gitlab-org/gitlab-ee!7276
parents d041e4f6 e73fc904
......@@ -63,7 +63,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
format.html do
usage_data_json = JSON.pretty_generate(Gitlab::UsageData.data)
render html: Gitlab::Highlight.highlight('payload.json', usage_data_json)
render html: Gitlab::Highlight.highlight('payload.json', usage_data_json, language: 'json')
end
format.json { render json: Gitlab::UsageData.to_json }
end
......
......@@ -92,7 +92,7 @@ class Projects::BlobController < Projects::ApplicationController
apply_diff_view_cookie!
@blob.load_all_data!
@lines = Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: @repository).lines
@lines = @blob.present.highlight.lines
@form = UnfoldForm.new(params)
......
# frozen_string_literal: true
module BlobHelper
def highlight(blob_name, blob_content, repository: nil, plain: false)
plain ||= blob_content.length > Blob::MAXIMUM_TEXT_HIGHLIGHT_SIZE
highlighted = Gitlab::Highlight.highlight(blob_name, blob_content, plain: plain, repository: repository)
def highlight(file_name, file_content, language: nil, plain: false)
highlighted = Gitlab::Highlight.highlight(file_name, file_content, plain: plain, language: language)
raw %(<pre class="code highlight"><code>#{highlighted}</code></pre>)
end
......
# frozen_string_literal: true
# Blob is a Rails-specific wrapper around Gitlab::Git::Blob objects
# Blob is a Rails-specific wrapper around Gitlab::Git::Blob, SnippetBlob and Ci::ArtifactBlob
class Blob < SimpleDelegator
prepend EE::Blob
include Presentable
include BlobLanguageFromGitAttributes
CACHE_TIME = 60 # Cache raw blobs referred to by a (mutable) ref for 1 minute
CACHE_TIME_IMMUTABLE = 3600 # Cache blobs referred to by an immutable reference for 1 hour
MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte
# Finding a viewer for a blob happens based only on extension and whether the
# blob is binary or text, which means 1 blob should only be matched by 1 viewer,
# and the order of these viewers doesn't really matter.
......@@ -123,10 +123,6 @@ class Blob < SimpleDelegator
end
end
def no_highlighting?
raw_size && raw_size > MAXIMUM_TEXT_HIGHLIGHT_SIZE
end
def empty?
raw_size == 0
end
......
# frozen_string_literal: true
# Applicable for blob classes with project attribute
module BlobLanguageFromGitAttributes
extend ActiveSupport::Concern
def language_from_gitattributes
return nil unless project
repository = project.repository
repository.gitattribute(path, 'gitlab-language')
end
end
# frozen_string_literal: true
class BlobPresenter < Gitlab::View::Presenter::Simple
presents :blob
def highlight(plain: nil)
blob.load_all_data! if blob.respond_to?(:load_all_data!)
Gitlab::Highlight.highlight(
blob.path,
blob.data,
language: blob.language_from_gitattributes,
plain: plain
)
end
end
......@@ -4,4 +4,6 @@
- blob.data.each_line.each_with_index do |_, index|
%span.diff-line-num= index + 1
.blob-content{ data: { blob_id: blob.id } }
= highlight(blob.path, blob.data, repository: nil, plain: blob.no_highlighting?)
%pre.code.highlight
%code
= blob.present.highlight
= render 'shared/file_highlight', blob: viewer.blob, repository: @repository
= render 'shared/file_highlight', blob: viewer.blob
......@@ -15,14 +15,14 @@
.col-md-2.text-center
Markdown
.col-md-10.code.js-syntax-highlight
= highlight('.md', badge.to_markdown)
= highlight('.md', badge.to_markdown, language: 'markdown')
.row
%hr
.row
.col-md-2.text-center
HTML
.col-md-10.code.js-syntax-highlight
= highlight('.html', badge.to_html)
= highlight('.html', badge.to_html, language: 'html')
.row
%hr
.row
......
......@@ -39,7 +39,7 @@
.blob-content
- snippet_chunks.each do |chunk|
- unless chunk[:data].empty?
= highlight(snippet.file_name, chunk[:data], repository: nil, plain: snippet.blob.no_highlighting?)
= highlight(snippet.file_name, chunk[:data])
- else
.file-content.code
.nothing-here-block Empty file
- repository = nil unless local_assigns.key?(:repository)
.file-content.code.js-syntax-highlight
.line-numbers
- if blob.data.present?
......@@ -13,4 +11,6 @@
= link_icon
= i
.blob-content{ data: { blob_id: blob.id } }
= highlight(blob.path, blob.data, repository: repository, plain: blob.no_highlighting?)
%pre.code.highlight
%code
= blob.present.highlight
......@@ -36,7 +36,7 @@
%li
.code.js-syntax-highlight.sherlock-code
:preserve
#{highlight("#{@query.id}.sql", @query.formatted_query)}
#{highlight("#{@query.id}.sql", @query.formatted_query, language: 'sql')}
.card
.card-header
......
......@@ -17,7 +17,7 @@
= t('sherlock.milliseconds')
%td
.code.js-syntax-highlight.sherlock-code
= highlight("#{query.id}.sql", query.formatted_query)
= highlight("#{query.id}.sql", query.formatted_query, language: 'sql')
%td
= link_to(t('sherlock.view'),
sherlock_transaction_query_path(@transaction, query),
......
......@@ -43,8 +43,7 @@ module Gitlab
def highlighted_lines
@blob.load_all_data!
@highlighted_lines ||=
Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines
@highlighted_lines ||= @blob.present.highlight.lines
end
def project
......
......@@ -3,6 +3,7 @@ module Gitlab
class File
include Gitlab::Routing
include IconsHelper
include Gitlab::Utils::StrongMemoize
CONTEXT_LINES = 3
......@@ -30,11 +31,8 @@ module Gitlab
end
def highlight_lines!
their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines
our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines
their_highlight = Gitlab::Highlight.highlight(their_path, their_lines, language: their_language).lines
our_highlight = Gitlab::Highlight.highlight(our_path, our_lines, language: our_language).lines
lines.each do |line|
line.rich_text =
......@@ -182,6 +180,34 @@ module Gitlab
raw_line[:line_new], parent_file: self)
end
end
def their_language
strong_memoize(:their_language) do
repository.gitattribute(their_path, 'gitlab-language')
end
end
def our_language
strong_memoize(:our_language) do
if our_path == their_path
their_language
else
repository.gitattribute(our_path, 'gitlab-language')
end
end
end
def their_lines
strong_memoize(:their_lines) do
lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
end
end
def our_lines
strong_memoize(:our_lines) do
lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
end
end
end
end
end
......@@ -79,7 +79,7 @@ module Gitlab
return [] unless blob
blob.load_all_data!
Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines
blob.present.highlight.lines
end
end
end
......
......@@ -4,22 +4,25 @@ module Gitlab
class Highlight
TIMEOUT_BACKGROUND = 30.seconds
TIMEOUT_FOREGROUND = 3.seconds
MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte
def self.highlight(blob_name, blob_content, repository: nil, plain: false)
new(blob_name, blob_content, repository: repository)
def self.highlight(blob_name, blob_content, language: nil, plain: false)
new(blob_name, blob_content, language: language)
.highlight(blob_content, continue: false, plain: plain)
end
attr_reader :blob_name
def initialize(blob_name, blob_content, repository: nil)
def initialize(blob_name, blob_content, language: nil)
@formatter = Rouge::Formatters::HTMLGitlab
@repository = repository
@language = language
@blob_name = blob_name
@blob_content = blob_content
end
def highlight(text, continue: true, plain: false)
plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE
highlighted_text = highlight_text(text, continue: continue, plain: plain)
highlighted_text = link_dependencies(text, highlighted_text) if blob_name
highlighted_text
......@@ -36,11 +39,9 @@ module Gitlab
private
def custom_language
language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language')
return nil unless language_name
return nil unless @language
Rouge::Lexer.find_fancy(language_name)
Rouge::Lexer.find_fancy(@language)
end
def highlight_text(text, continue: true, plain: false)
......
......@@ -82,7 +82,7 @@ module Gitlab
ref: ref,
startline: startline,
data: data.join,
project_id: project ? project.id : nil
project: project
)
end
......
......@@ -4,8 +4,10 @@ module Gitlab
class SearchResults
class FoundBlob
include EncodingHelper
include Presentable
include BlobLanguageFromGitAttributes
attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id
attr_reader :id, :filename, :basename, :ref, :startline, :data, :project
def initialize(opts = {})
@id = opts.fetch(:id, nil)
......@@ -15,6 +17,11 @@ module Gitlab
@startline = opts.fetch(:startline, nil)
@data = encode_utf8(opts.fetch(:data, nil))
@per_page = opts.fetch(:per_page, 20)
@project = opts.fetch(:project, nil)
# Some caller does not have project object (e.g. elastic search),
# yet they can trigger many calls in one go,
# causing duplicated queries.
# Allow those to just pass project_id instead.
@project_id = opts.fetch(:project_id, nil)
end
......@@ -22,8 +29,12 @@ module Gitlab
filename
end
def no_highlighting?
false
def project_id
@project_id || @project&.id
end
def present
super(presenter_class: BlobPresenter)
end
end
......
......@@ -2,7 +2,7 @@ require 'spec_helper'
describe 'User searches for wiki pages', :js do
let(:user) { create(:user) }
let(:project) { create(:project, :wiki_repo, namespace: user.namespace) }
let(:project) { create(:project, :repository, :wiki_repo, namespace: user.namespace) }
let!(:wiki_page) { create(:wiki_page, wiki: project.wiki, attrs: { title: 'test_wiki', content: 'Some Wiki content' }) }
before do
......
......@@ -3,63 +3,13 @@ require 'spec_helper'
describe BlobHelper do
include TreeHelper
let(:blob_name) { 'test.lisp' }
let(:no_context_content) { ":type \"assem\"))" }
let(:blob_content) { "(make-pathname :defaults name\n#{no_context_content}" }
let(:split_content) { blob_content.split("\n") }
let(:multiline_content) do
%q(
def test(input):
"""This is line 1 of a multi-line comment.
This is line 2.
"""
)
end
describe '#highlight' do
it 'returns plaintext for unknown lexer context' do
result = helper.highlight(blob_name, no_context_content)
expect(result).to eq(%[<pre class="code highlight"><code><span id="LC1" class="line" lang="">:type "assem"))</span></code></pre>])
it 'wraps highlighted content' do
expect(helper.highlight('test.rb', '52')).to eq(%q[<pre class="code highlight"><code><span id="LC1" class="line" lang="ruby"><span class="mi">52</span></span></code></pre>])
end
it 'returns plaintext for long blobs' do
stub_const('Blob::MAXIMUM_TEXT_HIGHLIGHT_SIZE', 1)
result = helper.highlight(blob_name, blob_content)
expect(result).to eq(%[<pre class="code highlight"><code><span id="LC1" class="line" lang="">(make-pathname :defaults name</span>\n<span id="LC2" class="line" lang="">:type "assem"))</span></code></pre>])
end
it 'highlights single block' do
expected = %Q[<pre class="code highlight"><code><span id="LC1" class="line" lang="common_lisp"><span class="p">(</span><span class="nb">make-pathname</span> <span class="ss">:defaults</span> <span class="nv">name</span></span>
<span id="LC2" class="line" lang="common_lisp"><span class="ss">:type</span> <span class="s">"assem"</span><span class="p">))</span></span></code></pre>]
expect(helper.highlight(blob_name, blob_content)).to eq(expected)
end
it 'highlights multi-line comments' do
result = helper.highlight(blob_name, multiline_content)
html = Nokogiri::HTML(result)
lines = html.search('.s')
expect(lines.count).to eq(3)
expect(lines[0].text).to eq('"""This is line 1 of a multi-line comment.')
expect(lines[1].text).to eq(' This is line 2.')
expect(lines[2].text).to eq(' """')
end
context 'diff highlighting' do
let(:blob_name) { 'test.diff' }
let(:blob_content) { "+aaa\n+bbb\n- ccc\n ddd\n"}
let(:expected) do
%q(<pre class="code highlight"><code><span id="LC1" class="line" lang="diff"><span class="gi">+aaa</span></span>
<span id="LC2" class="line" lang="diff"><span class="gi">+bbb</span></span>
<span id="LC3" class="line" lang="diff"><span class="gd">- ccc</span></span>
<span id="LC4" class="line" lang="diff"> ddd</span></code></pre>)
end
it 'highlights each line properly' do
result = helper.highlight(blob_name, blob_content)
expect(result).to eq(expected)
end
it 'handles plain version' do
expect(helper.highlight('test.rb', '52', plain: true)).to eq(%q[<pre class="code highlight"><code><span id="LC1" class="line" lang="">52</span></code></pre>])
end
end
......
......@@ -94,7 +94,7 @@ describe Gitlab::Git::AttributesParser, :seed_helper do
# It's a bit hard to test for something _not_ being processed. As such we'll
# just test the number of entries.
it 'ignores any comments and empty lines' do
expect(subject.patterns.length).to eq(10)
expect(subject.patterns.length).to eq(12)
end
end
......@@ -126,7 +126,7 @@ describe Gitlab::Git::AttributesParser, :seed_helper do
describe '#each_line' do
it 'iterates over every line in the attributes file' do
args = [String] * 14 # the number of lines in the file
args = [String] * 16 # the number of lines in the file
expect { |b| subject.each_line(&b) }.to yield_successive_args(*args)
end
......
......@@ -1194,6 +1194,34 @@ describe Gitlab::Git::Repository, :seed_helper do
end
end
describe '#gitattribute' do
let(:repository) { Gitlab::Git::Repository.new('default', TEST_GITATTRIBUTES_REPO_PATH, '') }
after do
ensure_seeds
end
it 'returns matching language attribute' do
expect(repository.gitattribute("custom-highlighting/test.gitlab-custom", 'gitlab-language')).to eq('ruby')
end
it 'returns matching language attribute with additional options' do
expect(repository.gitattribute("custom-highlighting/test.gitlab-cgi", 'gitlab-language')).to eq('erb?parent=json')
end
it 'returns nil if nothing matches' do
expect(repository.gitattribute("report.xslt", 'gitlab-language')).to eq(nil)
end
context 'without gitattributes file' do
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
it 'returns nil' do
expect(repository.gitattribute("README.md", 'gitlab-language')).to eq(nil)
end
end
end
describe '#ref_exists?' do
it 'returns true for an existing tag' do
expect(repository.ref_exists?('refs/heads/master')).to eq(true)
......
......@@ -5,44 +5,85 @@ describe Gitlab::Highlight do
let(:project) { create(:project, :repository) }
let(:repository) { project.repository }
let(:commit) { project.commit(sample_commit.id) }
describe 'custom highlighting from .gitattributes' do
let(:branch) { 'gitattributes' }
let(:blob) { repository.blob_at_branch(branch, path) }
describe 'language provided' do
let(:highlighter) do
described_class.new(blob.path, blob.data, repository: repository)
described_class.new('foo.erb', 'bar', language: 'erb?parent=json')
end
before do
project.change_head('gitattributes')
it 'sets correct lexer' do
expect(highlighter.lexer.tag).to eq 'erb'
expect(highlighter.lexer.parent.tag).to eq 'json'
end
end
describe 'basic language selection' do
let(:path) { 'custom-highlighting/test.gitlab-custom' }
it 'highlights as ruby' do
expect(highlighter.lexer.tag).to eq 'ruby'
end
describe '#highlight' do
let(:file_name) { 'test.lisp' }
let(:no_context_content) { ":type \"assem\"))" }
let(:content) { "(make-pathname :defaults name\n#{no_context_content}" }
let(:multiline_content) do
%q(
def test(input):
"""This is line 1 of a multi-line comment.
This is line 2.
"""
)
end
it 'highlights' do
expected = %Q[<span id="LC1" class="line" lang="common_lisp"><span class="p">(</span><span class="nb">make-pathname</span> <span class="ss">:defaults</span> <span class="nv">name</span></span>
<span id="LC2" class="line" lang="common_lisp"><span class="ss">:type</span> <span class="s">"assem"</span><span class="p">))</span></span>]
expect(described_class.highlight(file_name, content)).to eq(expected)
end
it 'returns plain version for unknown lexer context' do
result = described_class.highlight(file_name, no_context_content)
expect(result).to eq(%[<span id="LC1" class="line" lang="">:type "assem"))</span>])
end
describe 'cgi options' do
let(:path) { 'custom-highlighting/test.gitlab-cgi' }
it 'returns plain version for long content' do
stub_const('Gitlab::Highlight::MAXIMUM_TEXT_HIGHLIGHT_SIZE', 1)
result = described_class.highlight(file_name, content)
it 'highlights as json with erb' do
expect(highlighter.lexer.tag).to eq 'erb'
expect(highlighter.lexer.parent.tag).to eq 'json'
expect(result).to eq(%[<span id="LC1" class="line" lang="">(make-pathname :defaults name</span>\n<span id="LC2" class="line" lang="">:type "assem"))</span>])
end
it 'highlights multi-line comments' do
result = described_class.highlight(file_name, multiline_content)
html = Nokogiri::HTML(result)
lines = html.search('.s')
expect(lines.count).to eq(3)
expect(lines[0].text).to eq('"""This is line 1 of a multi-line comment.')
expect(lines[1].text).to eq(' This is line 2.')
expect(lines[2].text).to eq(' """')
end
context 'diff highlighting' do
let(:file_name) { 'test.diff' }
let(:content) { "+aaa\n+bbb\n- ccc\n ddd\n"}
let(:expected) do
%q(<span id="LC1" class="line" lang="diff"><span class="gi">+aaa</span></span>
<span id="LC2" class="line" lang="diff"><span class="gi">+bbb</span></span>
<span id="LC3" class="line" lang="diff"><span class="gd">- ccc</span></span>
<span id="LC4" class="line" lang="diff"> ddd</span>)
end
it 'highlights each line properly' do
result = described_class.highlight(file_name, content)
expect(result).to eq(expected)
end
end
end
describe '#highlight' do
describe 'with CRLF' do
let(:branch) { 'crlf-diff' }
let(:path) { 'files/whitespace' }
let(:blob) { repository.blob_at_branch(branch, path) }
let(:lines) do
described_class.highlight(blob.path, blob.data, repository: repository).lines
described_class.highlight(blob.path, blob.data).lines
end
it 'strips extra LFs' do
......
# frozen_string_literal: true
require 'spec_helper'
describe BlobLanguageFromGitAttributes do
include FakeBlobHelpers
let(:project) { build(:project, :repository) }
describe '#language_from_gitattributes' do
subject(:blob) { fake_blob(path: 'file.md') }
it 'returns return value from gitattribute' do
expect(blob.project.repository).to receive(:gitattribute).with(blob.path, 'gitlab-language').and_return('erb?parent=json')
expect(blob.language_from_gitattributes).to eq('erb?parent=json')
end
it 'returns nil if project is absent' do
allow(blob).to receive(:project).and_return(nil)
expect(blob.language_from_gitattributes).to eq(nil)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe BlobPresenter, :seed_helper do
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') }
let(:git_blob) do
Gitlab::Git::Blob.find(
repository,
'fa1b1e6c004a68b7d8763b86455da9e6b23e36d6',
'files/ruby/regex.rb'
)
end
let(:blob) { Blob.new(git_blob) }
describe '#highlight' do
subject { described_class.new(blob) }
it 'returns highlighted content' do
expect(Gitlab::Highlight).to receive(:highlight).with('files/ruby/regex.rb', git_blob.data, plain: nil, language: nil)
subject.highlight
end
it 'returns plain content when :plain is true' do
expect(Gitlab::Highlight).to receive(:highlight).with('files/ruby/regex.rb', git_blob.data, plain: true, language: nil)
subject.highlight(plain: true)
end
context 'gitlab-language contains a match' do
before do
allow(blob).to receive(:language_from_gitattributes).and_return('ruby')
end
it 'passes language to inner call' do
expect(Gitlab::Highlight).to receive(:highlight).with('files/ruby/regex.rb', git_blob.data, plain: nil, language: 'ruby')
subject.highlight
end
end
end
end
# frozen_string_literal: true
require_relative 'test_env'
# This file is specific to specs in spec/lib/gitlab/git/
SEED_STORAGE_PATH = TestEnv.repos_path
TEST_REPO_PATH = 'gitlab-git-test.git'.freeze
TEST_NORMAL_REPO_PATH = 'not-bare-repo.git'.freeze
TEST_MUTABLE_REPO_PATH = 'mutable-repo.git'.freeze
TEST_BROKEN_REPO_PATH = 'broken-repo.git'.freeze
TEST_REPO_PATH = 'gitlab-git-test.git'
TEST_NORMAL_REPO_PATH = 'not-bare-repo.git'
TEST_MUTABLE_REPO_PATH = 'mutable-repo.git'
TEST_BROKEN_REPO_PATH = 'broken-repo.git'
TEST_GITATTRIBUTES_REPO_PATH = 'with-git-attributes.git'
module SeedHelper
GITLAB_GIT_TEST_REPO_URL = File.expand_path('../gitlab-git-test.git', __dir__).freeze
GITLAB_GIT_TEST_REPO_URL = File.expand_path('../gitlab-git-test.git', __dir__)
def ensure_seeds
if File.exist?(SEED_STORAGE_PATH)
......@@ -66,6 +69,11 @@ module SeedHelper
end
def create_git_attributes
system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --bare #{TEST_REPO_PATH} #{TEST_GITATTRIBUTES_REPO_PATH}),
chdir: SEED_STORAGE_PATH,
out: '/dev/null',
err: '/dev/null')
dir = File.join(SEED_STORAGE_PATH, 'with-git-attributes.git', 'info')
FileUtils.mkdir_p(dir)
......@@ -82,6 +90,8 @@ foo/bar.* foo
*.cgi key=value?p1=v1&p2=v2
/*.png gitlab-language=png
*.binary binary
/custom-highlighting/*.gitlab-custom gitlab-language=ruby
/custom-highlighting/*.gitlab-cgi gitlab-language=erb?parent=json
# This uses a tab instead of spaces to ensure the parser also supports this.
*.md\tgitlab-language=markdown
......
......@@ -29,6 +29,8 @@ describe 'projects/blob/_viewer.html.haml' do
controller.params[:namespace_id] = project.namespace.to_param
controller.params[:project_id] = project.to_param
controller.params[:id] = File.join('master', blob.path)
allow(project.repository).to receive(:gitattribute).and_return(nil)
end
def render_view
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment