Commit 838097a4 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'master' into 8-3-stable

parents 5fcb4a14 a5a9cd30
......@@ -54,6 +54,7 @@ v 8.3.0 (unreleased)
- Suppress warning about missing `.gitlab-ci.yml` if builds are disabled
- Do not show build status unless builds are enabled and `.gitlab-ci.yml` is present
- Persist runners registration token in database
- Fix online editor should not remove newlines at the end of the file
v 8.2.3
- Fix application settings cache not expiring after changes (Stan Hu)
......@@ -61,8 +62,6 @@ v 8.2.3
- Update documentation for "Guest" permissions
- Properly convert Emoji-only comments into Award Emojis
- Enable devise paranoid mode to prevent user enumeration attack
v 8.2.3
- Webhook payload has an added, modified and removed properties for each commit
- Fix 500 error when creating a merge request that removes a submodule
......
This diff is collapsed.
......@@ -3,7 +3,6 @@
.panel-heading {
padding: 7px $gl-padding;
line-height: 42px !important;
}
.panel-body {
......@@ -15,3 +14,7 @@
}
}
}
.container-blank .panel .panel-heading {
line-height: 42px !important;
}
......@@ -19,7 +19,7 @@ $border-color: #dce0e6;
$table-border-color: #eef0f2;
$background-color: #F7F8FA;
$header-height: 58px;
$fixed-layout-width: 1200px;
$fixed-layout-width: 1280px;
$gl-gray: #7f8fa4;
$gl-padding: 16px;
$gl-avatar-size: 46px;
......
......@@ -13,7 +13,8 @@ class Projects::NotesController < Projects::ApplicationController
@notes.each do |note|
notes_json[:notes] << {
id: note.id,
html: note_to_html(note)
html: note_to_html(note),
valid: note.valid?
}
end
......
......@@ -20,7 +20,7 @@ module GitlabMarkdownHelper
end
user = current_user if defined?(current_user)
gfm_body = Gitlab::Markdown.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
gfm_body = Banzai.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
if fragment.children.size == 1 && fragment.children[0].name == 'a'
......@@ -50,7 +50,7 @@ module GitlabMarkdownHelper
context[:project] ||= @project
html = Gitlab::Markdown.render(text, context)
html = Banzai.render(text, context)
context.merge!(
current_user: (current_user if defined?(current_user)),
......@@ -61,7 +61,7 @@ module GitlabMarkdownHelper
ref: @ref
)
Gitlab::Markdown.post_process(html, context)
Banzai.post_process(html, context)
end
def asciidoc(text)
......
......@@ -121,6 +121,6 @@ module IssuesHelper
end
end
# Required for Gitlab::Markdown::IssueReferenceFilter
# Required for Banzai::Filter::IssueReferenceFilter
module_function :url_for_issue
end
......@@ -107,6 +107,6 @@ module LabelsHelper
options_from_collection_for_select(grouped_labels, 'name', 'title', params[:label_name])
end
# Required for Gitlab::Markdown::LabelReferenceFilter
# Required for Banzai::Filter::LabelReferenceFilter
module_function :render_colored_label, :text_color_for_bg, :escape_once
end
......@@ -23,7 +23,7 @@ module Mentionable
included do
if self < Participable
participant ->(current_user) { mentioned_users(current_user, load_lazy_references: false) }
participant ->(current_user) { mentioned_users(current_user) }
end
end
......@@ -43,9 +43,9 @@ module Mentionable
self
end
def all_references(current_user = self.author, text = nil, load_lazy_references: true)
ext = Gitlab::ReferenceExtractor.new(self.project, current_user, load_lazy_references: load_lazy_references)
def all_references(current_user = self.author, text = nil)
ext = Gitlab::ReferenceExtractor.new(self.project, current_user)
if text
ext.analyze(text)
else
......@@ -59,13 +59,13 @@ module Mentionable
ext
end
def mentioned_users(current_user = nil, load_lazy_references: true)
all_references(current_user, load_lazy_references: load_lazy_references).users
def mentioned_users(current_user = nil)
all_references(current_user).users
end
# Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
def referenced_mentionables(current_user = self.author, text = nil, load_lazy_references: true)
refs = all_references(current_user, text, load_lazy_references: load_lazy_references)
def referenced_mentionables(current_user = self.author, text = nil)
refs = all_references(current_user, text)
refs = (refs.issues + refs.merge_requests + refs.commits)
# We're using this method instead of Array diffing because that requires
......
......@@ -38,20 +38,21 @@ module Participable
# Be aware that this method makes a lot of sql queries.
# Save result into variable if you are going to reuse it inside same request
def participants(current_user = self.author, load_lazy_references: true)
participants = self.class.participant_attrs.flat_map do |attr|
value =
if attr.respond_to?(:call)
instance_exec(current_user, &attr)
else
send(attr)
end
participants =
Gitlab::ReferenceExtractor.lazily do
self.class.participant_attrs.flat_map do |attr|
value =
if attr.respond_to?(:call)
instance_exec(current_user, &attr)
else
send(attr)
end
participants_for(value, current_user)
end.compact.uniq
if load_lazy_references
participants = Gitlab::Markdown::ReferenceFilter::LazyReference.load(participants).uniq
participants_for(value, current_user)
end.compact.uniq
end
unless Gitlab::ReferenceExtractor.lazy?
participants.select! do |user|
user.can?(:read_project, project)
end
......@@ -64,12 +65,12 @@ module Participable
def participants_for(value, current_user = nil)
case value
when User, Gitlab::Markdown::ReferenceFilter::LazyReference
when User, Banzai::LazyReference
[value]
when Enumerable, ActiveRecord::Relation
value.flat_map { |v| participants_for(v, current_user) }
when Participable
value.participants(current_user, load_lazy_references: false)
value.participants(current_user)
end
end
end
......@@ -84,11 +84,11 @@ class Issue < ActiveRecord::Base
end
def referenced_merge_requests
references = [self, *notes].flat_map do |note|
note.all_references(load_lazy_references: false).merge_requests
end.uniq
Gitlab::Markdown::ReferenceFilter::LazyReference.load(references).uniq.sort_by(&:iid)
Gitlab::ReferenceExtractor.lazily do
[self, *notes].flat_map do |note|
note.all_references(load_lazy_references: false).merge_requests
end
end.sort_by(&:iid)
end
# Reset issue events cache
......
......@@ -373,11 +373,11 @@ class Note < ActiveRecord::Base
end
def contains_emoji_only?
note =~ /\A#{Gitlab::Markdown::EmojiFilter.emoji_pattern}\s?\Z/
note =~ /\A#{Banzai::Filter::EmojiFilter.emoji_pattern}\s?\Z/
end
def award_emoji_name
original_name = note.match(Gitlab::Markdown::EmojiFilter.emoji_pattern)[1]
original_name = note.match(Banzai::Filter::EmojiFilter.emoji_pattern)[1]
AwardEmoji.normilize_emoji_name(original_name)
end
end
......@@ -16,8 +16,7 @@
= select_tag :encoding, options_for_select([ "base64", "text" ], "text"), class: 'select2'
.file-content.code
%pre.js-edit-mode-pane#editor
= params[:content] || local_assigns[:blob_data]
%pre.js-edit-mode-pane#editor #{params[:content] || local_assigns[:blob_data]}
- if local_assigns[:path]
.js-edit-mode-pane#preview.hide
.center
......
......@@ -7,4 +7,4 @@
= render "projects/notes/form", view: diff_view
:javascript
new Notes("#{namespace_project_notes_path(namespace_id: @project.namespace, target_id: @noteable.id, target_type: @noteable.class.name.underscore)}", #{@notes.map(&:id).to_json}, #{Time.now.to_i}, "#{diff_view}")
var notes = new Notes("#{namespace_project_notes_path(namespace_id: @project.namespace, target_id: @noteable.id, target_type: @noteable.class.name.underscore)}", #{@notes.map(&:id).to_json}, #{Time.now.to_i}, "#{diff_view}")
# Rake tasks
- [Backup restore](backup_restore.md)
- [Check](check.md)
- [Cleanup](cleanup.md)
- [Features](features.md)
- [Maintenance](maintenance.md) and self-checks
- [User management](user_management.md)
- [Web hooks](web_hooks.md)
- [Import](import.md) of git repositories in bulk
- [Rebuild authorized_keys file](http://doc.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators
\ No newline at end of file
- [Rebuild authorized_keys file](http://doc.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators
# Check Rake Tasks
## Repository Integrity
Even though Git is very resilient and tries to prevent data integrity issues,
there are times when things go wrong. The following Rake tasks intend to
help GitLab administrators diagnose problem repositories so they can be fixed.
There are 3 things that are checked to determine integrity.
1. Git repository file system check ([git fsck](https://git-scm.com/docs/git-fsck)).
This step verifies the connectivity and validity of objects in the repository.
1. Check for `config.lock` in the repository directory.
1. Check for any branch/references lock files in `refs/heads`.
It's important to note that the existence of `config.lock` or reference locks
alone do not necessarily indicate a problem. Lock files are routinely created
and removed as Git and GitLab perform operations on the repository. They serve
to prevent data integrity issues. However, if a Git operation is interrupted these
locks may not be cleaned up properly.
The following symptoms may indicate a problem with repository integrity. If users
experience these symptoms you may use the rake tasks described below to determine
exactly which repositories are causing the trouble.
- Receiving an error when trying to push code - `remote: error: cannot lock ref`
- A 500 error when viewing the GitLab dashboard or when accessing a specific project.
### Check all GitLab repositories
This task loops through all repositories on the GitLab server and runs the
3 integrity checks described previously.
```
# omnibus-gitlab
sudo gitlab-rake gitlab:repo:check
# installation from source
bundle exec rake gitlab:repo:check RAILS_ENV=production
```
### Check repositories for a specific user
This task checks all repositories that a specific user has access to. This is important
because sometimes you know which user is experiencing trouble but you don't know
which project might be the cause.
If the rake task is executed without brackets at the end, you will be prompted
to enter a username.
```bash
# omnibus-gitlab
sudo gitlab-rake gitlab:user:check_repos
sudo gitlab-rake gitlab:user:check_repos[<username>]
# installation from source
bundle exec rake gitlab:user:check_repos RAILS_ENV=production
bundle exec rake gitlab:user:check_repos[<username>] RAILS_ENV=production
```
Example output:
![gitlab:user:check_repos output](check_repos_output.png)
......@@ -197,3 +197,9 @@ Feature: Project Issues
And I should not see labels field
And I submit new issue "500 error on profile"
Then I should see issue "500 error on profile"
@javascript
Scenario: Another user adds a comment to issue I'm currently viewing
Given I visit issue page "Release 0.4"
And another user adds a comment with text "Yay!" to issue "Release 0.4"
Then I should see a new comment with text "Yay!"
......@@ -34,6 +34,17 @@ Feature: Project Source Browse Files
Then I am redirected to the new file
And I should see its new content
@javascript
Scenario: I can create and commit file with new lines at the end of file
Given I click on "New file" link in repo
And I edit code with new lines at end of file
And I fill the new file name
And I fill the commit message
And I click on "Commit Changes"
Then I am redirected to the new file
And I click button "Edit"
And I should see its content with new lines preserved at end of file
@javascript
Scenario: I can upload file and commit
Given I click on "Upload file" link in repo
......
......@@ -284,6 +284,16 @@ class Spinach::Features::ProjectIssues < Spinach::FeatureSteps
end
end
step 'another user adds a comment with text "Yay!" to issue "Release 0.4"' do
issue = Issue.find_by!(title: 'Release 0.4')
create(:note_on_issue, noteable: issue, note: 'Yay!')
end
step 'I should see a new comment with text "Yay!"' do
page.within '#notes' do
expect(page).to have_content('Yay!')
end
end
def filter_issue(text)
fill_in 'issue_search', with: text
end
......
......@@ -37,6 +37,10 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps
expect(page).to have_content new_gitignore_content
end
step 'I should see its content with new lines preserved at end of file' do
expect(evaluate_script('blob.editor.getValue()')).to eq "Sample\n\n\n"
end
step 'I click link "Raw"' do
click_link 'Raw'
end
......@@ -62,6 +66,10 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps
set_new_content
end
step 'I edit code with new lines at end of file' do
execute_script('blob.editor.setValue("Sample\n\n\n")')
end
step 'I fill the new file name' do
fill_in :file_name, with: new_file_name
end
......
......@@ -2,7 +2,7 @@ require 'spinach/capybara'
require 'capybara/poltergeist'
# Give CI some extra time
timeout = (ENV['CI'] || ENV['CI_SERVER']) ? 90 : 10
timeout = (ENV['CI'] || ENV['CI_SERVER']) ? 90 : 15
Capybara.javascript_driver = :poltergeist
Capybara.register_driver :poltergeist do |app|
......
module Banzai
def self.render(text, context = {})
Renderer.render(text, context)
end
def self.render_result(text, context = {})
Renderer.render_result(text, context)
end
def self.post_process(html, context)
Renderer.post_process(html, context)
end
end
require 'banzai'
module Banzai
# Common methods for ReferenceFilters that support an optional cross-project
# reference.
module CrossProjectReference
# Given a cross-project reference string, get the Project record
#
# Defaults to value of `context[:project]` if:
# * No reference is given OR
# * Reference given doesn't exist
#
# ref - String reference.
#
# Returns a Project, or nil if the reference can't be found
def project_from_ref(ref)
return context[:project] unless ref
Project.find_with_namespace(ref)
end
end
end
require 'active_support/core_ext/string/output_safety'
require 'banzai'
module Banzai
module Filter
def self.[](name)
const_get("#{name.to_s.camelize}Filter")
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# Issues, Merge Requests, Snippets, Commits and Commit Ranges share
# similar functionality in reference filtering.
class AbstractReferenceFilter < ReferenceFilter
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
require 'uri'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML Filter for auto-linking URLs in HTML.
#
# Based on HTML::Pipeline::AutolinkFilter
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces commit range references with links.
#
# This filter supports cross-project references.
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces commit references with links.
#
# This filter supports cross-project references.
......
require 'action_controller'
require 'gitlab/markdown'
require 'banzai'
require 'gitlab_emoji'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces :emoji: with images.
#
# Based on HTML::Pipeline::EmojiFilter
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces external issue tracker references with links.
# References are ignored if the project doesn't use an external issue
# tracker.
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML Filter to add a `rel="nofollow"` attribute to external links
#
class ExternalLinkFilter < HTML::Pipeline::Filter
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces issue references with links. References to
# issues that do not exist are ignored.
#
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces label references with links.
class LabelReferenceFilter < ReferenceFilter
# Public: Find label references in text
......
module Gitlab
module Markdown
require 'banzai'
require 'html/pipeline/filter'
module Banzai
module Filter
class MarkdownFilter < HTML::Pipeline::TextFilter
def initialize(text, context = nil, result = nil)
super text, context, result
......@@ -11,8 +14,8 @@ module Gitlab
html.rstrip!
html
end
private
private
def self.redcarpet_options
# https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces merge request references with links. References
# to merge requests that do not exist are ignored.
#
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that removes references to records that the current user does
# not have permission to view.
#
......@@ -27,7 +27,7 @@ module Gitlab
def user_can_reference?(node)
if node.has_attribute?('data-reference-filter')
reference_type = node.attr('data-reference-filter')
reference_filter = Gitlab::Markdown.const_get(reference_type)
reference_filter = Banzai::Filter.const_get(reference_type)
reference_filter.user_can_reference?(current_user, node, context)
else
......
require 'active_support/core_ext/string/output_safety'
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# Base class for GitLab Flavored Markdown reference filters.
#
# References within <pre>, <code>, <a>, and <style> elements are ignored.
......@@ -12,27 +12,6 @@ module Gitlab
# :project (required) - Current project, ignored if reference is cross-project.
# :only_path - Generate path-only links.
class ReferenceFilter < HTML::Pipeline::Filter
LazyReference = Struct.new(:klass, :ids) do
def self.load(refs)
lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
ids = refs.flat_map(&:ids)
klass.where(id: ids)
end
values + lazy_values
end
def load
self.klass.where(id: self.ids)
end
end
def self.[](name)
Markdown.const_get("#{name.to_s.camelize}ReferenceFilter")
end
def self.user_can_reference?(user, node, context)
if node.has_attribute?('data-project')
project_id = node.attr('data-project').to_i
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that gathers all referenced records that the current user has
# permission to view.
#
......@@ -20,7 +20,7 @@ module Gitlab
gather_references(node)
end
load_lazy_references unless context[:load_lazy_references] == false
load_lazy_references unless ReferenceExtractor.lazy?
doc
end
......@@ -31,7 +31,7 @@ module Gitlab
return unless node.has_attribute?('data-reference-filter')
reference_type = node.attr('data-reference-filter')
reference_filter = Gitlab::Markdown.const_get(reference_type)
reference_filter = Banzai::Filter.const_get(reference_type)
return if context[:reference_filter] && reference_filter != context[:reference_filter]
......@@ -47,11 +47,10 @@ module Gitlab
end
end
# Will load all references of one type using one query.
def load_lazy_references
refs = result[:references]
refs.each do |type, values|
refs[type] = ReferenceFilter::LazyReference.load(values)
refs[type] = ReferenceExtractor.lazily(values)
end
end
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
require 'uri'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that "fixes" relative links to files in a repository.
#
# Context options:
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
require 'html/pipeline/sanitization_filter'
module Gitlab
module Markdown
module Banzai
module Filter
# Sanitize HTML
#
# Extends HTML::Pipeline::SanitizationFilter with a custom whitelist.
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces snippet references with links. References to
# snippets that do not exist are ignored.
#
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
require 'rouge/plugins/redcarpet'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML Filter to highlight fenced code blocks
#
class SyntaxHighlightFilter < HTML::Pipeline::Filter
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that adds an anchor child element to all Headers in a
# document, so that they can be linked to.
#
......
require 'gitlab/markdown'
require 'banzai'
require 'task_list/filter'
module Gitlab
module Markdown
module Banzai
module Filter
# Work around a bug in the default TaskList::Filter that adds a `task-list`
# class to every list element, regardless of whether or not it contains a
# task list.
......
require 'gitlab/markdown'
require 'banzai'
require 'html/pipeline/filter'
require 'uri'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that "fixes" relative upload links to files.
# Context options:
# :project (required) - Current project
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Filter
# HTML filter that replaces user or group references with links.
#
# A special `@all` reference is also supported.
......
require 'banzai'
module Banzai
class LazyReference
def self.load(refs)
lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
ids = refs.flat_map(&:ids)
klass.where(id: ids)
end
values + lazy_values
end
attr_reader :klass, :ids
def initialize(klass, ids)
@klass = klass
@ids = Array.wrap(ids).map(&:to_i)
end
def load
self.klass.where(id: self.ids)
end
end
end
require 'banzai'
module Banzai
module Pipeline
def self.[](name)
name ||= :full
const_get("#{name.to_s.camelize}Pipeline")
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
class AsciidocPipeline < Pipeline
module Banzai
module Pipeline
class AsciidocPipeline < BasePipeline
def self.filters
[
Gitlab::Markdown::RelativeLinkFilter
Filter::RelativeLinkFilter
]
end
end
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class AtomPipeline < FullPipeline
def self.transform_context(context)
super(context).merge(
only_path: false,
xhtml: true
super(context).merge(
only_path: false,
xhtml: true
)
end
end
......
require 'banzai'
require 'html/pipeline'
module Banzai
module Pipeline
class BasePipeline
def self.filters
[]
end
def self.transform_context(context)
context
end
def self.html_pipeline
@html_pipeline ||= HTML::Pipeline.new(filters)
end
class << self
%i(call to_document to_html).each do |meth|
define_method(meth) do |text, context|
context = transform_context(context)
html_pipeline.send(meth, text, context)
end
end
end
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
module CombinedPipeline
def self.new(*pipelines)
Class.new(Pipeline) do
Class.new(BasePipeline) do
const_set :PIPELINES, pipelines
def self.pipelines
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class DescriptionPipeline < FullPipeline
def self.transform_context(context)
super(context).merge(
super(context).merge(
# SanitizationFilter
inline_sanitization: true
)
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class EmailPipeline < FullPipeline
def self.transform_context(context)
super(context).merge(
super(context).merge(
only_path: false
)
end
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
end
......
require 'banzai'
module Banzai
module Pipeline
class GfmPipeline < BasePipeline
def self.filters
@filters ||= [
Filter::SyntaxHighlightFilter,
Filter::SanitizationFilter,
Filter::UploadLinkFilter,
Filter::EmojiFilter,
Filter::TableOfContentsFilter,
Filter::AutolinkFilter,
Filter::ExternalLinkFilter,
Filter::UserReferenceFilter,
Filter::IssueReferenceFilter,
Filter::ExternalIssueReferenceFilter,
Filter::MergeRequestReferenceFilter,
Filter::SnippetReferenceFilter,
Filter::CommitRangeReferenceFilter,
Filter::CommitReferenceFilter,
Filter::LabelReferenceFilter,
Filter::TaskListFilter
]
end
def self.transform_context(context)
context.merge(
only_path: true,
# EmojiFilter
asset_host: Gitlab::Application.config.asset_host,
asset_root: Gitlab.config.gitlab.base_url
)
end
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class NotePipeline < FullPipeline
def self.transform_context(context)
super(context).merge(
super(context).merge(
# TableOfContentsFilter
no_header_anchors: true
)
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
class PlainMarkdownPipeline < Pipeline
module Banzai
module Pipeline
class PlainMarkdownPipeline < BasePipeline
def self.filters
[
Gitlab::Markdown::MarkdownFilter
Filter::MarkdownFilter
]
end
end
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
class PostProcessPipeline < Pipeline
module Banzai
module Pipeline
class PostProcessPipeline < BasePipeline
def self.filters
[
Gitlab::Markdown::RelativeLinkFilter,
Gitlab::Markdown::RedactorFilter
Filter::RelativeLinkFilter,
Filter::RedactorFilter
]
end
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
class ReferenceExtractionPipeline < Pipeline
module Banzai
module Pipeline
class ReferenceExtractionPipeline < BasePipeline
def self.filters
[
Gitlab::Markdown::ReferenceGathererFilter
Filter::ReferenceGathererFilter
]
end
end
......
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
module Banzai
module Pipeline
class SingleLinePipeline < GfmPipeline
end
end
end
require 'banzai'
module Banzai
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor
class << self
LAZY_KEY = :banzai_reference_extractor_lazy
def lazy?
Thread.current[LAZY_KEY]
end
def lazily(values = nil, &block)
return (values || block.call).uniq if lazy?
begin
Thread.current[LAZY_KEY] = true
values ||= block.call
Banzai::LazyReference.load(values.uniq).uniq
ensure
Thread.current[LAZY_KEY] = false
end
end
end
def initialize
@texts = []
end
def analyze(text, context = {})
@texts << Renderer.render(text, context)
end
def references(type, context = {})
filter = Banzai::Filter["#{type}_reference"]
context.merge!(
pipeline: :reference_extraction,
# ReferenceGathererFilter
reference_filter: filter
)
self.class.lazily do
@texts.flat_map do |html|
text_context = context.dup
result = Renderer.render_result(html, text_context)
result[:references][type]
end.uniq
end
end
end
end
require 'html/pipeline'
module Gitlab
# Custom parser for GitLab-flavored Markdown
#
# See the files in `lib/gitlab/markdown/` for specific processing information.
module Markdown
module Banzai
module Renderer
# Convert a Markdown String into an HTML-safe String of HTML
#
# Note that while the returned HTML will have been sanitized of dangerous
......@@ -75,41 +70,7 @@ module Gitlab
def self.full_cache_key(cache_key, pipeline_name)
return unless cache_key
["markdown", *cache_key, pipeline_name || :full]
["banzai", *cache_key, pipeline_name || :full]
end
# Provide autoload paths for filters to prevent a circular dependency error
autoload :AutolinkFilter, 'gitlab/markdown/filter/autolink_filter'
autoload :CommitRangeReferenceFilter, 'gitlab/markdown/filter/commit_range_reference_filter'
autoload :CommitReferenceFilter, 'gitlab/markdown/filter/commit_reference_filter'
autoload :EmojiFilter, 'gitlab/markdown/filter/emoji_filter'
autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/filter/external_issue_reference_filter'
autoload :ExternalLinkFilter, 'gitlab/markdown/filter/external_link_filter'
autoload :IssueReferenceFilter, 'gitlab/markdown/filter/issue_reference_filter'
autoload :LabelReferenceFilter, 'gitlab/markdown/filter/label_reference_filter'
autoload :MarkdownFilter, 'gitlab/markdown/filter/markdown_filter'
autoload :MergeRequestReferenceFilter, 'gitlab/markdown/filter/merge_request_reference_filter'
autoload :RedactorFilter, 'gitlab/markdown/filter/redactor_filter'
autoload :ReferenceGathererFilter, 'gitlab/markdown/filter/reference_gatherer_filter'
autoload :RelativeLinkFilter, 'gitlab/markdown/filter/relative_link_filter'
autoload :SanitizationFilter, 'gitlab/markdown/filter/sanitization_filter'
autoload :SnippetReferenceFilter, 'gitlab/markdown/filter/snippet_reference_filter'
autoload :SyntaxHighlightFilter, 'gitlab/markdown/filter/syntax_highlight_filter'
autoload :TableOfContentsFilter, 'gitlab/markdown/filter/table_of_contents_filter'
autoload :TaskListFilter, 'gitlab/markdown/filter/task_list_filter'
autoload :UserReferenceFilter, 'gitlab/markdown/filter/user_reference_filter'
autoload :UploadLinkFilter, 'gitlab/markdown/filter/upload_link_filter'
autoload :AsciidocPipeline, 'gitlab/markdown/pipeline/asciidoc_pipeline'
autoload :AtomPipeline, 'gitlab/markdown/pipeline/atom_pipeline'
autoload :DescriptionPipeline, 'gitlab/markdown/pipeline/description_pipeline'
autoload :EmailPipeline, 'gitlab/markdown/pipeline/email_pipeline'
autoload :FullPipeline, 'gitlab/markdown/pipeline/full_pipeline'
autoload :GfmPipeline, 'gitlab/markdown/pipeline/gfm_pipeline'
autoload :NotePipeline, 'gitlab/markdown/pipeline/note_pipeline'
autoload :PlainMarkdownPipeline, 'gitlab/markdown/pipeline/plain_markdown_pipeline'
autoload :PostProcessPipeline, 'gitlab/markdown/pipeline/post_process_pipeline'
autoload :ReferenceExtractionPipeline, 'gitlab/markdown/pipeline/reference_extraction_pipeline'
autoload :SingleLinePipeline, 'gitlab/markdown/pipeline/single_line_pipeline'
end
end
......@@ -32,7 +32,7 @@ module Gitlab
html = ::Asciidoctor.convert(input, asciidoc_opts)
if context[:project]
html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc))
html = Banzai.render(html, context.merge(pipeline: :asciidoc))
end
html.html_safe
......
require 'gitlab/markdown'
module Gitlab
module Markdown
# Common methods for ReferenceFilters that support an optional cross-project
# reference.
module CrossProjectReference
# Given a cross-project reference string, get the Project record
#
# Defaults to value of `context[:project]` if:
# * No reference is given OR
# * Reference given doesn't exist
#
# ref - String reference.
#
# Returns a Project, or nil if the reference can't be found
def project_from_ref(ref)
return context[:project] unless ref
Project.find_with_namespace(ref)
end
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
module Markdown
class Pipeline
def self.[](name)
name ||= :full
Markdown.const_get("#{name.to_s.camelize}Pipeline")
const_get("#{name.to_s.camelize}Pipeline")
end
def self.filters
......
require 'gitlab/markdown'
module Gitlab
module Markdown
class GfmPipeline < Pipeline
def self.filters
@filters ||= [
Gitlab::Markdown::SyntaxHighlightFilter,
Gitlab::Markdown::SanitizationFilter,
Gitlab::Markdown::UploadLinkFilter,
Gitlab::Markdown::EmojiFilter,
Gitlab::Markdown::TableOfContentsFilter,
Gitlab::Markdown::AutolinkFilter,
Gitlab::Markdown::ExternalLinkFilter,
Gitlab::Markdown::UserReferenceFilter,
Gitlab::Markdown::IssueReferenceFilter,
Gitlab::Markdown::ExternalIssueReferenceFilter,
Gitlab::Markdown::MergeRequestReferenceFilter,
Gitlab::Markdown::SnippetReferenceFilter,
Gitlab::Markdown::CommitRangeReferenceFilter,
Gitlab::Markdown::CommitReferenceFilter,
Gitlab::Markdown::LabelReferenceFilter,
Gitlab::Markdown::TaskListFilter
]
end
def self.transform_context(context)
context.merge(
only_path: true,
# EmojiFilter
asset_host: Gitlab::Application.config.asset_host,
asset_root: Gitlab.config.gitlab.base_url
)
end
end
end
end
require 'gitlab/markdown'
require 'banzai'
module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor
attr_accessor :project, :current_user, :load_lazy_references
class ReferenceExtractor < Banzai::ReferenceExtractor
attr_accessor :project, :current_user
def initialize(project, current_user = nil, load_lazy_references: true)
def initialize(project, current_user = nil)
@project = project
@current_user = current_user
@load_lazy_references = load_lazy_references
@texts = []
@references = {}
super()
end
def analyze(text, options = {})
@texts << Gitlab::Markdown.render(text, options.merge(project: project))
def analyze(text, context = {})
super(text, context.merge(project: project))
end
%i(user label issue merge_request snippet commit commit_range).each do |type|
define_method("#{type}s") do
@references[type] ||= pipeline_result(type)
@references[type] ||= references(type, project: project, current_user: current_user)
end
end
private
# Instantiate and call HTML::Pipeline with a single reference filter type,
# returning the result
#
# filter_type - Symbol reference type (e.g., :commit, :issue, etc.)
#
# Returns the results Array for the requested filter type
def pipeline_result(filter_type)
filter = Gitlab::Markdown::ReferenceFilter[filter_type]
context = {
pipeline: :reference_extraction,
project: project,
current_user: current_user,
# ReferenceGathererFilter
load_lazy_references: false,
reference_filter: filter
}
values = @texts.flat_map do |html|
text_context = context.dup
result = Gitlab::Markdown.render_result(html, text_context)
result[:references][filter_type]
end.uniq
if @load_lazy_references
values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq
end
values
end
end
end
......@@ -822,10 +822,27 @@ namespace :gitlab do
namespace_dirs.each do |namespace_dir|
repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
repo_dirs.each do |dir|
puts "\nChecking repo at #{dir}"
system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir)
end
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
end
end
end
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
task :check_repos, [:username] => :environment do |t, args|
username = args[:username] || prompt("Check repository integrity for which username? ".blue)
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
File.join(
Gitlab.config.gitlab_shell.repos_path,
"#{p.path_with_namespace}.git"
)
end
repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
else
puts "\nUser '#{username}' not found".red
end
end
end
......@@ -952,4 +969,35 @@ namespace :gitlab do
false
end
end
def check_repo_integrity(repo_dir)
puts "\nChecking repo at #{repo_dir.yellow}"
git_fsck(repo_dir)
check_config_lock(repo_dir)
check_ref_locks(repo_dir)
end
def git_fsck(repo_dir)
puts "Running `git fsck`".yellow
system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
end
def check_config_lock(repo_dir)
config_exists = File.exist?(File.join(repo_dir,'config.lock'))
config_output = config_exists ? 'yes'.red : 'no'.green
puts "'config.lock' file exists?".yellow + " ... #{config_output}"
end
def check_ref_locks(repo_dir)
lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock'))
if lock_files.present?
puts "Ref lock files exist:".red
lock_files.each do |lock_file|
puts " #{lock_file}"
end
else
puts "No ref lock files exist".green
end
end
end
require 'spec_helper'
describe Gitlab::Markdown::ReferenceFilter, benchmark: true do
describe Banzai::Filter::ReferenceFilter, benchmark: true do
let(:input) do
html = <<-EOF
<p>Hello @alice and @bob, how are you doing today?</p>
......
require 'spec_helper'
feature 'Issue notes polling' do
let!(:project) { create(:project, :public) }
let!(:issue) { create(:issue, project: project) }
background do
visit namespace_project_issue_path(project.namespace, project, issue)
end
scenario 'Another user adds a comment to an issue', js: true do
note = create(:note_on_issue, noteable: issue, note: 'Looks good!')
page.execute_script('notes.refresh();')
expect(page).to have_selector("#note_#{note.id}", text: 'Looks good!')
end
end
require 'spec_helper'
describe Gitlab::Markdown::CrossProjectReference, lib: true do
describe Banzai::CrossProjectReference, lib: true do
include described_class
describe '#project_from_ref' do
......
require 'spec_helper'
describe Gitlab::Markdown::AutolinkFilter, lib: true do
describe Banzai::Filter::AutolinkFilter, lib: true do
include FilterSpecHelper
let(:link) { 'http://about.gitlab.com/' }
......
require 'spec_helper'
describe Gitlab::Markdown::CommitRangeReferenceFilter, lib: true do
describe Banzai::Filter::CommitRangeReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:project, :public) }
......
require 'spec_helper'
describe Gitlab::Markdown::CommitReferenceFilter, lib: true do
describe Banzai::Filter::CommitReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:project, :public) }
......
require 'spec_helper'
describe Gitlab::Markdown::EmojiFilter, lib: true do
describe Banzai::Filter::EmojiFilter, lib: true do
include FilterSpecHelper
before do
......
require 'spec_helper'
describe Gitlab::Markdown::ExternalIssueReferenceFilter, lib: true do
describe Banzai::Filter::ExternalIssueReferenceFilter, lib: true do
include FilterSpecHelper
def helper
......
require 'spec_helper'
describe Gitlab::Markdown::ExternalLinkFilter, lib: true do
describe Banzai::Filter::ExternalLinkFilter, lib: true do
include FilterSpecHelper
it 'ignores elements without an href attribute' do
......
require 'spec_helper'
describe Gitlab::Markdown::IssueReferenceFilter, lib: true do
describe Banzai::Filter::IssueReferenceFilter, lib: true do
include FilterSpecHelper
def helper
......
require 'spec_helper'
require 'html/pipeline'
describe Gitlab::Markdown::LabelReferenceFilter, lib: true do
describe Banzai::Filter::LabelReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:empty_project, :public) }
......
require 'spec_helper'
describe Gitlab::Markdown::MergeRequestReferenceFilter, lib: true do
describe Banzai::Filter::MergeRequestReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:project, :public) }
......
require 'spec_helper'
describe Gitlab::Markdown::RedactorFilter, lib: true do
describe Banzai::Filter::RedactorFilter, lib: true do
include ActionView::Helpers::UrlHelper
include FilterSpecHelper
......
require 'spec_helper'
describe Gitlab::Markdown::ReferenceGathererFilter, lib: true do
describe Banzai::Filter::ReferenceGathererFilter, lib: true do
include ActionView::Helpers::UrlHelper
include FilterSpecHelper
......
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::Markdown::RelativeLinkFilter, lib: true do
describe Banzai::Filter::RelativeLinkFilter, lib: true do
def filter(doc, contexts = {})
contexts.reverse_merge!({
commit: project.commit,
......
require 'spec_helper'
describe Gitlab::Markdown::SanitizationFilter, lib: true do
describe Banzai::Filter::SanitizationFilter, lib: true do
include FilterSpecHelper
describe 'default whitelist' do
......
require 'spec_helper'
describe Gitlab::Markdown::SnippetReferenceFilter, lib: true do
describe Banzai::Filter::SnippetReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:empty_project, :public) }
......
require 'spec_helper'
describe Gitlab::Markdown::SyntaxHighlightFilter, lib: true do
describe Banzai::Filter::SyntaxHighlightFilter, lib: true do
include FilterSpecHelper
it 'highlights valid code blocks' do
......
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::Markdown::TableOfContentsFilter, lib: true do
describe Banzai::Filter::TableOfContentsFilter, lib: true do
include FilterSpecHelper
def header(level, text)
......
require 'spec_helper'
describe Gitlab::Markdown::TaskListFilter, lib: true do
describe Banzai::Filter::TaskListFilter, lib: true do
include FilterSpecHelper
it 'does not apply `task-list` class to non-task lists' do
......
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::Markdown::UploadLinkFilter, lib: true do
describe Banzai::Filter::UploadLinkFilter, lib: true do
def filter(doc, contexts = {})
contexts.reverse_merge!({
project: project
......
require 'spec_helper'
describe Gitlab::Markdown::UserReferenceFilter, lib: true do
describe Banzai::Filter::UserReferenceFilter, lib: true do
include FilterSpecHelper
let(:project) { create(:empty_project, :public) }
......
......@@ -50,7 +50,7 @@ module Gitlab
filtered_html = '<b>ASCII</b>'
allow(Asciidoctor).to receive(:convert).and_return(html)
expect(Gitlab::Markdown).to receive(:render)
expect(Banzai).to receive(:render)
.with(html, context.merge(pipeline: :asciidoc))
.and_return(filtered_html)
......
# Helper methods for Gitlab::Markdown filter specs
# Helper methods for Banzai filter specs
#
# Must be included into specs manually
module FilterSpecHelper
......@@ -10,49 +10,49 @@ module FilterSpecHelper
# if none is provided.
#
# html - HTML String to pass to the filter's `call` method.
# contexts - Hash context for the filter. (default: {project: project})
# context - Hash context for the filter. (default: {project: project})
#
# Returns a Nokogiri::XML::DocumentFragment
def filter(html, contexts = {})
def filter(html, context = {})
if defined?(project)
contexts.reverse_merge!(project: project)
context.reverse_merge!(project: project)
end
described_class.call(html, contexts)
described_class.call(html, context)
end
# Run text through HTML::Pipeline with the current filter and return the
# result Hash
#
# body - String text to run through the pipeline
# contexts - Hash context for the filter. (default: {project: project})
# context - Hash context for the filter. (default: {project: project})
#
# Returns the Hash
def pipeline_result(body, contexts = {})
contexts.reverse_merge!(project: project) if defined?(project)
def pipeline_result(body, context = {})
context.reverse_merge!(project: project) if defined?(project)
pipeline = HTML::Pipeline.new([described_class], contexts)
pipeline = HTML::Pipeline.new([described_class], context)
pipeline.call(body)
end
def reference_pipeline(contexts = {})
contexts.reverse_merge!(project: project) if defined?(project)
def reference_pipeline(context = {})
context.reverse_merge!(project: project) if defined?(project)
filters = [
Gitlab::Markdown::AutolinkFilter,
Banzai::Filter::AutolinkFilter,
described_class,
Gitlab::Markdown::ReferenceGathererFilter
Banzai::Filter::ReferenceGathererFilter
]
HTML::Pipeline.new(filters, contexts)
HTML::Pipeline.new(filters, context)
end
def reference_pipeline_result(body, contexts = {})
reference_pipeline(contexts).call(body)
def reference_pipeline_result(body, context = {})
reference_pipeline(context).call(body)
end
def reference_filter(html, contexts = {})
reference_pipeline(contexts).to_document(html)
def reference_filter(html, context = {})
reference_pipeline(context).to_document(html)
end
# Modify a String reference to make it invalid
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment