Commit 05985b49 authored by James Lopez's avatar James Lopez

Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into fix/project-import_url

parents e5f7a545 447a9050
...@@ -34,6 +34,7 @@ v 8.7.0 (unreleased) ...@@ -34,6 +34,7 @@ v 8.7.0 (unreleased)
- Fix a bug whith trailing slash in bamboo_url - Fix a bug whith trailing slash in bamboo_url
- Add links to CI setup documentation from project settings and builds pages - Add links to CI setup documentation from project settings and builds pages
- Handle nil descriptions in Slack issue messages (Stan Hu) - Handle nil descriptions in Slack issue messages (Stan Hu)
- Add automated repository integrity checks
- API: Expose open_issues_count, closed_issues_count, open_merge_requests_count for labels (Robert Schilling) - API: Expose open_issues_count, closed_issues_count, open_merge_requests_count for labels (Robert Schilling)
- API: Ability to star and unstar a project (Robert Schilling) - API: Ability to star and unstar a project (Robert Schilling)
- Add default scope to projects to exclude projects pending deletion - Add default scope to projects to exclude projects pending deletion
...@@ -71,6 +72,7 @@ v 8.6.5 (unreleased) ...@@ -71,6 +72,7 @@ v 8.6.5 (unreleased)
- Improved markdown forms - Improved markdown forms
- Diffs load at the correct point when linking from from number - Diffs load at the correct point when linking from from number
- Selected diff rows highlight - Selected diff rows highlight
- Fix emoji catgories in the emoji picker
v 8.6.6 v 8.6.6
- Fix error on language detection when repository has no HEAD (e.g., master branch). !3654 (Jeroen Bobbeldijk) - Fix error on language detection when repository has no HEAD (e.g., master branch). !3654 (Jeroen Bobbeldijk)
......
...@@ -2,7 +2,7 @@ class @Subscription ...@@ -2,7 +2,7 @@ class @Subscription
constructor: (container) -> constructor: (container) ->
$container = $(container) $container = $(container)
@url = $container.attr('data-url') @url = $container.attr('data-url')
@subscribe_button = $container.find('.subscribe-button') @subscribe_button = $container.find('.js-subscribe-button')
@subscription_status = $container.find('.subscription-status') @subscription_status = $container.find('.subscription-status')
@subscribe_button.unbind('click').click(@toggleSubscription) @subscribe_button.unbind('click').click(@toggleSubscription)
......
...@@ -67,6 +67,24 @@ ...@@ -67,6 +67,24 @@
line-height: $code_line_height; line-height: $code_line_height;
font-size: $code_font_size; font-size: $code_font_size;
&.noteable_line {
position: relative;
&.old {
&:before {
content: '-';
position: absolute;
}
}
&.new {
&:before {
content: '+';
position: absolute;
}
}
}
span { span {
white-space: pre; white-space: pre;
} }
...@@ -391,3 +409,23 @@ ...@@ -391,3 +409,23 @@
margin-bottom: 0; margin-bottom: 0;
} }
} }
.file-holder {
.diff-line-num:not(.js-unfold-bottom) {
a {
&:before {
content: attr(data-linenumber);
}
}
}
}
.discussion {
.diff-content {
.diff-line-num {
&:before {
content: attr(data-linenumber);
}
}
}
}
...@@ -173,12 +173,6 @@ ...@@ -173,12 +173,6 @@
} }
} }
.subscribe-button {
span {
margin-top: 0;
}
}
&.right-sidebar-collapsed { &.right-sidebar-collapsed {
/* Extra small devices (phones, less than 768px) */ /* Extra small devices (phones, less than 768px) */
display: none; display: none;
......
...@@ -117,7 +117,7 @@ ...@@ -117,7 +117,7 @@
padding: 6px; padding: 6px;
color: $gl-text-color; color: $gl-text-color;
&.subscribe-button { &.label-subscribe-button {
padding-left: 0; padding-left: 0;
} }
} }
......
...@@ -276,8 +276,7 @@ ul.notes { ...@@ -276,8 +276,7 @@ ul.notes {
.diff-file tr.line_holder { .diff-file tr.line_holder {
@mixin show-add-diff-note { @mixin show-add-diff-note {
filter: alpha(opacity=100); display: inline-block;
opacity: 1.0;
} }
.add-diff-note { .add-diff-note {
...@@ -291,13 +290,8 @@ ul.notes { ...@@ -291,13 +290,8 @@ ul.notes {
position: absolute; position: absolute;
z-index: 10; z-index: 10;
width: 32px; width: 32px;
transition: all 0.2s ease;
// "hide" it by default // "hide" it by default
opacity: 0.0; display: none;
filter: alpha(opacity=0);
&:hover { &:hover {
background: $gl-info; background: $gl-info;
color: #fff; color: #fff;
......
...@@ -19,6 +19,15 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -19,6 +19,15 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
redirect_to admin_runners_path redirect_to admin_runners_path
end end
def clear_repository_check_states
RepositoryCheck::ClearWorker.perform_async
redirect_to(
admin_application_settings_path,
notice: 'Started asynchronous removal of all repository check states.'
)
end
private private
def set_application_setting def set_application_setting
...@@ -82,6 +91,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -82,6 +91,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:akismet_enabled, :akismet_enabled,
:akismet_api_key, :akismet_api_key,
:email_author_in_body, :email_author_in_body,
:repository_checks_enabled,
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [] import_sources: []
) )
......
class Admin::ProjectsController < Admin::ApplicationController class Admin::ProjectsController < Admin::ApplicationController
before_action :project, only: [:show, :transfer] before_action :project, only: [:show, :transfer, :repository_check]
before_action :group, only: [:show, :transfer] before_action :group, only: [:show, :transfer]
def index def index
...@@ -8,6 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController ...@@ -8,6 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController
@projects = @projects.where("projects.visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present? @projects = @projects.where("projects.visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present?
@projects = @projects.with_push if params[:with_push].present? @projects = @projects.with_push if params[:with_push].present?
@projects = @projects.abandoned if params[:abandoned].present? @projects = @projects.abandoned if params[:abandoned].present?
@projects = @projects.where(last_repository_check_failed: true) if params[:last_repository_check_failed].present?
@projects = @projects.non_archived unless params[:with_archived].present? @projects = @projects.non_archived unless params[:with_archived].present?
@projects = @projects.search(params[:name]) if params[:name].present? @projects = @projects.search(params[:name]) if params[:name].present?
@projects = @projects.sort(@sort = params[:sort]) @projects = @projects.sort(@sort = params[:sort])
...@@ -30,6 +31,15 @@ class Admin::ProjectsController < Admin::ApplicationController ...@@ -30,6 +31,15 @@ class Admin::ProjectsController < Admin::ApplicationController
redirect_to admin_namespace_project_path(@project.namespace, @project) redirect_to admin_namespace_project_path(@project.namespace, @project)
end end
def repository_check
RepositoryCheck::SingleRepositoryWorker.perform_async(@project.id)
redirect_to(
admin_namespace_project_path(@project.namespace, @project),
notice: 'Repository check was triggered.'
)
end
protected protected
def project def project
......
...@@ -14,7 +14,7 @@ class ApplicationController < ActionController::Base ...@@ -14,7 +14,7 @@ class ApplicationController < ActionController::Base
before_action :check_password_expiration before_action :check_password_expiration
before_action :check_2fa_requirement before_action :check_2fa_requirement
before_action :ldap_security_check before_action :ldap_security_check
before_action :sentry_user_context before_action :sentry_context
before_action :default_headers before_action :default_headers
before_action :add_gon_variables before_action :add_gon_variables
before_action :configure_permitted_parameters, if: :devise_controller? before_action :configure_permitted_parameters, if: :devise_controller?
...@@ -41,13 +41,15 @@ class ApplicationController < ActionController::Base ...@@ -41,13 +41,15 @@ class ApplicationController < ActionController::Base
protected protected
def sentry_user_context def sentry_context
if Rails.env.production? && current_application_settings.sentry_enabled && current_user if Rails.env.production? && current_application_settings.sentry_enabled
if current_user
Raven.user_context( Raven.user_context(
id: current_user.id, id: current_user.id,
email: current_user.email, email: current_user.email,
username: current_user.username, username: current_user.username,
) )
end
Raven.tags_context(program: sentry_program_context) Raven.tags_context(program: sentry_program_context)
end end
......
...@@ -40,6 +40,7 @@ class GroupsController < Groups::ApplicationController ...@@ -40,6 +40,7 @@ class GroupsController < Groups::ApplicationController
@last_push = current_user.recent_push if current_user @last_push = current_user.recent_push if current_user
@projects = @projects.includes(:namespace) @projects = @projects.includes(:namespace)
@projects = @projects.sorted_by_activity
@projects = filter_projects(@projects) @projects = filter_projects(@projects)
@projects = @projects.sort(@sort = params[:sort]) @projects = @projects.sort(@sort = params[:sort])
@projects = @projects.page(params[:page]) if params[:filter_projects].blank? @projects = @projects.page(params[:page]) if params[:filter_projects].blank?
......
...@@ -40,10 +40,11 @@ module DiffHelper ...@@ -40,10 +40,11 @@ module DiffHelper
(unfold) ? 'unfold js-unfold' : '' (unfold) ? 'unfold js-unfold' : ''
end end
def diff_line_content(line) def diff_line_content(line, line_type = nil)
if line.blank? if line.blank?
" &nbsp;".html_safe " &nbsp;".html_safe
else else
line[0] = ' ' if %w[new old].include?(line_type)
line line
end end
end end
......
class RepositoryCheckMailer < BaseMailer
def notify(failed_count)
if failed_count == 1
@message = "One project failed its last repository check"
else
@message = "#{failed_count} projects failed their last repository check"
end
mail(
to: User.admins.pluck(:email),
subject: @message
)
end
end
...@@ -153,7 +153,8 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -153,7 +153,8 @@ class ApplicationSetting < ActiveRecord::Base
require_two_factor_authentication: false, require_two_factor_authentication: false,
two_factor_grace_period: 48, two_factor_grace_period: 48,
recaptcha_enabled: false, recaptcha_enabled: false,
akismet_enabled: false akismet_enabled: false,
repository_checks_enabled: true,
) )
end end
......
...@@ -797,7 +797,7 @@ class Repository ...@@ -797,7 +797,7 @@ class Repository
def search_files(query, ref) def search_files(query, ref)
offset = 2 offset = 2
args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -e #{query} #{ref || root_ref}) args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -e #{Regexp.escape(query)} #{ref || root_ref})
Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/) Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
end end
......
...@@ -271,5 +271,24 @@ ...@@ -271,5 +271,24 @@
.col-sm-10 .col-sm-10
= f.text_field :sentry_dsn, class: 'form-control' = f.text_field :sentry_dsn, class: 'form-control'
%fieldset
%legend Repository Checks
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
= f.label :repository_checks_enabled do
= f.check_box :repository_checks_enabled
Enable Repository Checks
.help-block
GitLab will periodically run
%a{ href: 'https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html', target: 'blank' } 'git fsck'
in all project and wiki repositories to look for silent disk corruption issues.
.form-group
.col-sm-offset-2.col-sm-10
= link_to 'Clear all repository checks', clear_repository_check_states_admin_application_settings_path, data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
.help-block
If you got a lot of false alarms from repository checks you can choose to clear all repository check information from the database.
.form-actions .form-actions
= f.submit 'Save', class: 'btn btn-save' = f.submit 'Save', class: 'btn btn-save'
- page_title "Logs" - page_title "Logs"
- loggers = [Gitlab::GitLogger, Gitlab::AppLogger, - loggers = [Gitlab::GitLogger, Gitlab::AppLogger,
Gitlab::ProductionLogger, Gitlab::SidekiqLogger] Gitlab::ProductionLogger, Gitlab::SidekiqLogger,
Gitlab::RepositoryCheckLogger]
%ul.nav-links.log-tabs %ul.nav-links.log-tabs
- loggers.each do |klass| - loggers.each do |klass|
%li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
.row.prepend-top-default .row.prepend-top-default
%aside.col-md-3 %aside.col-md-3
.admin-filter .panel.admin-filter
= form_tag admin_namespaces_projects_path, method: :get, class: '' do = form_tag admin_namespaces_projects_path, method: :get, class: '' do
.form-group .form-group
= label_tag :name, 'Name:' = label_tag :name, 'Name:'
...@@ -38,7 +38,13 @@ ...@@ -38,7 +38,13 @@
%span.descr %span.descr
= visibility_level_icon(level) = visibility_level_icon(level)
= label = label
%hr %fieldset
%strong Problems
.checkbox
= label_tag :last_repository_check_failed do
= check_box_tag :last_repository_check_failed, 1, params[:last_repository_check_failed]
%span Last repository check failed
= hidden_field_tag :sort, params[:sort] = hidden_field_tag :sort, params[:sort]
= button_tag "Search", class: "btn submit btn-primary" = button_tag "Search", class: "btn submit btn-primary"
= link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel" = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel"
......
...@@ -5,6 +5,16 @@ ...@@ -5,6 +5,16 @@
%i.fa.fa-pencil-square-o %i.fa.fa-pencil-square-o
Edit Edit
%hr %hr
- if @project.last_repository_check_failed?
.row
.col-md-12
.panel
.panel-heading.alert.alert-danger
Last repository check
= "(#{time_ago_in_words(@project.last_repository_check_at)} ago)"
failed. See
= link_to 'repocheck.log', admin_logs_path
for error messages.
.row .row
.col-md-6 .col-md-6
.panel.panel-default .panel.panel-default
...@@ -95,6 +105,32 @@ ...@@ -95,6 +105,32 @@
.col-sm-offset-2.col-sm-10 .col-sm-offset-2.col-sm-10
= f.submit 'Transfer', class: 'btn btn-primary' = f.submit 'Transfer', class: 'btn btn-primary'
.panel.panel-default.repository-check
.panel-heading
Repository check
.panel-body
= form_for @project, url: repository_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
.form-group
- if @project.last_repository_check_at.nil?
This repository has never been checked.
- else
This repository was last checked
= @project.last_repository_check_at.to_s(:medium) + '.'
The check
- if @project.last_repository_check_failed?
= succeed '.' do
%strong.cred failed
See
= link_to 'repocheck.log', admin_logs_path
for error messages.
- else
passed.
= link_to icon('question-circle'), help_page_path('administration', 'repository_checks')
.form-group
= f.submit 'Trigger repository check', class: 'btn btn-primary'
.col-md-6 .col-md-6
- if @group - if @group
.panel.panel-default .panel.panel-default
......
- if @lines.present? - if @lines.present?
- if @form.unfold? && @form.since != 1 && !@form.bottom? - if @form.unfold? && @form.since != 1 && !@form.bottom?
%tr.line_holder{ id: @form.since } %tr.line_holder{ id: @form.since }
= render "projects/diffs/match_line", {line: @match_line, = render "projects/diffs/match_line", { line: @match_line,
line_old: @form.since, line_new: @form.since, bottom: false, new_file: false} line_old: @form.since, line_new: @form.since, bottom: false, new_file: false }
- @lines.each_with_index do |line, index| - @lines.each_with_index do |line, index|
- line_new = index + @form.since - line_new = index + @form.since
- line_old = line_new - @form.offset - line_old = line_new - @form.offset
%tr.line_holder %tr.line_holder
%td.old_line.diff-line-num{data: {linenumber: line_old}} %td.old_line.diff-line-num{ data: { linenumber: line_old } }
= link_to raw(line_old), "#" = link_to raw(line_old), "#"
%td.new_line.diff-line-num %td.new_line.diff-line-num{ data: { linenumber: line_old } }
= link_to raw(line_new) , "#" = link_to raw(line_new) , "#"
%td.line_content.noteable_line==#{' ' * @form.indent}#{line} %td.line_content.noteable_line==#{' ' * @form.indent}#{line}
- if @form.unfold? && @form.bottom? && @form.to < @blob.loc - if @form.unfold? && @form.bottom? && @form.to < @blob.loc
%tr.line_holder{ id: @form.to } %tr.line_holder{ id: @form.to }
= render "projects/diffs/match_line", {line: @match_line, = render "projects/diffs/match_line", { line: @match_line,
line_old: @form.to, line_new: @form.to, bottom: true, new_file: false} line_old: @form.to, line_new: @form.to, bottom: true, new_file: false }
- type = line.type - type = line.type
%tr.line_holder{id: line_code, class: type} %tr.line_holder{ id: line_code, class: type }
- case type - case type
- when 'match' - when 'match'
= render "projects/diffs/match_line", {line: line.text, = render "projects/diffs/match_line", { line: line.text,
line_old: line.old_pos, line_new: line.new_pos, bottom: false, new_file: diff_file.new_file} line_old: line.old_pos, line_new: line.new_pos, bottom: false, new_file: diff_file.new_file }
- when 'nonewline' - when 'nonewline'
%td.old_line.diff-line-num %td.old_line.diff-line-num
%td.new_line.diff-line-num %td.new_line.diff-line-num
%td.line_content.match= line.text %td.line_content.match= line.text
- else - else
%td.old_line.diff-line-num{class: type} %td.old_line.diff-line-num{ class: type, data: { linenumber: line.new_pos } }
- link_text = raw(type == "new" ? "&nbsp;" : line.old_pos) - link_text = type == "new" ? "&nbsp;".html_safe : line.old_pos
- if defined?(plain) && plain - if defined?(plain) && plain
= link_text = link_text
- else - else
= link_to link_text, "##{line_code}", id: line_code = link_to "", "##{line_code}", id: line_code, data: { linenumber: link_text }
- if @comments_allowed && can?(current_user, :create_note, @project) - if @comments_allowed && can?(current_user, :create_note, @project)
= link_to_new_diff_note(line_code) = link_to_new_diff_note(line_code)
%td.new_line.diff-line-num{class: type, data: {linenumber: line.new_pos}} %td.new_line.diff-line-num{ class: type, data: { linenumber: line.new_pos } }
- link_text = raw(type == "old" ? "&nbsp;" : line.new_pos) - link_text = type == "old" ? "&nbsp;".html_safe : line.new_pos
- if defined?(plain) && plain - if defined?(plain) && plain
= link_text = link_text
- else - else
= link_to link_text, "##{line_code}", id: line_code = link_to "", "##{line_code}", id: line_code, data: { linenumber: link_text }
%td.line_content{class: "noteable_line #{type} #{line_code}", data: { line_code: line_code }}= diff_line_content(line.text) %td.line_content{ class: ['noteable_line', type, line_code], data: { line_code: line_code } }= diff_line_content(line.text, type)
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
.label-subscription{data: {url: toggle_subscription_namespace_project_label_path(@project.namespace, @project, label)}} .label-subscription{data: {url: toggle_subscription_namespace_project_label_path(@project.namespace, @project, label)}}
.subscription-status{data: {status: label_subscription_status(label)}} .subscription-status{data: {status: label_subscription_status(label)}}
%a.subscribe-button.btn.action-buttons{data: {toggle: "tooltip"}} %button.js-subscribe-button.label-subscribe-button.btn.action-buttons{ type: "button", data: { toggle: "tooltip" } }
%span= label_subscription_toggle_button_text(label) %span= label_subscription_toggle_button_text(label)
- if can? current_user, :admin_label, @project - if can? current_user, :admin_label, @project
......
...@@ -20,11 +20,9 @@ ...@@ -20,11 +20,9 @@
%td.new_line.diff-line-num= "..." %td.new_line.diff-line-num= "..."
%td.line_content.match= line.text %td.line_content.match= line.text
- else - else
%td.old_line.diff-line-num %td.old_line.diff-line-num{ data: { linenumber: type == "new" ? "&nbsp;".html_safe : line.old_pos } }
= raw(type == "new" ? "&nbsp;" : line.old_pos) %td.new_line.diff-line-num{ data: { linenumber: type == "old" ? "&nbsp;".html_safe : line.new_pos } }
%td.new_line.diff-line-num %td.line_content{ class: ['noteable_line', type, line_code], line_code: line_code }= diff_line_content(line.text, type)
= raw(type == "old" ? "&nbsp;" : line.new_pos)
%td.line_content{class: "noteable_line #{type} #{line_code}", line_code: line_code}= diff_line_content(line.text)
- if line_code == note.line_code - if line_code == note.line_code
= render "projects/notes/diff_notes_with_reply", notes: discussion_notes = render "projects/notes/diff_notes_with_reply", notes: discussion_notes
%p
#{@message}.
%p
= link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repository_check_failed: 1)
#{@message}.
\
View details: #{admin_namespaces_projects_url(last_repository_check_failed: 1)}
...@@ -128,7 +128,7 @@ ...@@ -128,7 +128,7 @@
.title.hide-collapsed .title.hide-collapsed
Notifications Notifications
- subscribtion_status = subscribed ? 'subscribed' : 'unsubscribed' - subscribtion_status = subscribed ? 'subscribed' : 'unsubscribed'
%button.btn.btn-block.btn-gray.subscribe-button.hide-collapsed{:type => 'button'} %button.btn.btn-block.btn-gray.js-subscribe-button.issuable-subscribe-button.hide-collapsed{ type: "button" }
%span= subscribed ? 'Unsubscribe' : 'Subscribe' %span= subscribed ? 'Unsubscribe' : 'Subscribe'
.subscription-status.hide-collapsed{data: {status: subscribtion_status}} .subscription-status.hide-collapsed{data: {status: subscribtion_status}}
.unsubscribed{class: ( 'hidden' if subscribed )} .unsubscribed{class: ( 'hidden' if subscribed )}
......
class AdminEmailWorker
include Sidekiq::Worker
sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
def perform
repository_check_failed_count = Project.where(last_repository_check_failed: true).count
return if repository_check_failed_count.zero?
RepositoryCheckMailer.notify(repository_check_failed_count).deliver_now
end
end
module RepositoryCheck
class BatchWorker
include Sidekiq::Worker
RUN_TIME = 3600
sidekiq_options retry: false
def perform
start = Time.now
# This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new
# RepositoryCheckWorker each hour so that as long as there are repositories to
# check, only one (or two) will be checked at a time.
project_ids.each do |project_id|
break if Time.now - start >= RUN_TIME
break unless current_settings.repository_checks_enabled
next unless try_obtain_lease(project_id)
SingleRepositoryWorker.new.perform(project_id)
end
end
private
# Project.find_each does not support WHERE clauses and
# Project.find_in_batches does not support ordering. So we just build an
# array of ID's. This is OK because we do it only once an hour, because
# getting ID's from Postgres is not terribly slow, and because no user
# has to sit and wait for this query to finish.
def project_ids
limit = 10_000
never_checked_projects = Project.where('last_repository_check_at IS NULL').limit(limit).
pluck(:id)
old_check_projects = Project.where('last_repository_check_at < ?', 1.month.ago).
reorder('last_repository_check_at ASC').limit(limit).pluck(:id)
never_checked_projects + old_check_projects
end
def try_obtain_lease(id)
# Use a 24-hour timeout because on servers/projects where 'git fsck' is
# super slow we definitely do not want to run it twice in parallel.
Gitlab::ExclusiveLease.new(
"project_repository_check:#{id}",
timeout: 24.hours
).try_obtain
end
def current_settings
# No caching of the settings! If we cache them and an admin disables
# this feature, an active RepositoryCheckWorker would keep going for up
# to 1 hour after the feature was disabled.
if Rails.env.test?
Gitlab::CurrentSettings.fake_application_settings
else
ApplicationSetting.current
end
end
end
end
module RepositoryCheck
class ClearWorker
include Sidekiq::Worker
sidekiq_options retry: false
def perform
# Do small batched updates because these updates will be slow and locking
Project.select(:id).find_in_batches(batch_size: 100) do |batch|
Project.where(id: batch.map(&:id)).update_all(
last_repository_check_failed: nil,
last_repository_check_at: nil,
)
end
end
end
end
module RepositoryCheck
class SingleRepositoryWorker
include Sidekiq::Worker
sidekiq_options retry: false
def perform(project_id)
project = Project.find(project_id)
project.update_columns(
last_repository_check_failed: !check(project),
last_repository_check_at: Time.now,
)
end
private
def check(project)
# Use 'map do', not 'all? do', to prevent short-circuiting
[project.repository, project.wiki.repository].map do |repository|
git_fsck(repository.path_to_repo)
end.all?
end
def git_fsck(path)
cmd = %W(nice git --git-dir=#{path} fsck)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
true
else
Gitlab::RepositoryCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
false
end
end
end
end
...@@ -164,6 +164,13 @@ production: &base ...@@ -164,6 +164,13 @@ production: &base
# Flag stuck CI builds as failed # Flag stuck CI builds as failed
stuck_ci_builds_worker: stuck_ci_builds_worker:
cron: "0 0 * * *" cron: "0 0 * * *"
# Periodically run 'git fsck' on all repositories. If started more than
# once per hour you will have concurrent 'git fsck' jobs.
repository_check_worker:
cron: "20 * * * *"
# Send admin emails once a day
admin_email_worker:
cron: "0 0 * * *"
# Remove outdated repository archives # Remove outdated repository archives
repository_archive_cache_worker: repository_archive_cache_worker:
......
...@@ -241,11 +241,16 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) ...@@ -241,11 +241,16 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker'
Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
# #
# GitLab Shell # GitLab Shell
# #
......
...@@ -264,6 +264,7 @@ Rails.application.routes.draw do ...@@ -264,6 +264,7 @@ Rails.application.routes.draw do
member do member do
put :transfer put :transfer
post :repository_check
end end
resources :runner_projects resources :runner_projects
...@@ -281,6 +282,7 @@ Rails.application.routes.draw do ...@@ -281,6 +282,7 @@ Rails.application.routes.draw do
resource :application_settings, only: [:show, :update] do resource :application_settings, only: [:show, :update] do
resources :services resources :services
put :reset_runners_token put :reset_runners_token
put :clear_repository_check_states
end end
resources :labels resources :labels
......
class ProjectAddRepositoryCheck < ActiveRecord::Migration
def change
add_column :projects, :last_repository_check_failed, :boolean
add_index :projects, :last_repository_check_failed
add_column :projects, :last_repository_check_at, :datetime
end
end
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
# #
class MigrateNewNotificationSetting < ActiveRecord::Migration class MigrateNewNotificationSetting < ActiveRecord::Migration
def up def up
timestamp = Time.now timestamp = Time.now.strftime('%F %T')
execute "INSERT INTO notification_settings ( user_id, source_id, source_type, level, created_at, updated_at ) SELECT user_id, source_id, source_type, notification_level, '#{timestamp}', '#{timestamp}' FROM members WHERE user_id IS NOT NULL" execute "INSERT INTO notification_settings ( user_id, source_id, source_type, level, created_at, updated_at ) SELECT user_id, source_id, source_type, notification_level, '#{timestamp}', '#{timestamp}' FROM members WHERE user_id IS NOT NULL"
end end
......
class AddRepositoryChecksEnabledSetting < ActiveRecord::Migration
def change
add_column :application_settings, :repository_checks_enabled, :boolean, default: true
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20160331223143) do ActiveRecord::Schema.define(version: 20160412140240) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -77,6 +77,7 @@ ActiveRecord::Schema.define(version: 20160331223143) do ...@@ -77,6 +77,7 @@ ActiveRecord::Schema.define(version: 20160331223143) do
t.string "akismet_api_key" t.string "akismet_api_key"
t.boolean "email_author_in_body", default: false t.boolean "email_author_in_body", default: false
t.integer "default_group_visibility" t.integer "default_group_visibility"
t.boolean "repository_checks_enabled", default: true
end end
create_table "audit_events", force: :cascade do |t| create_table "audit_events", force: :cascade do |t|
...@@ -746,6 +747,8 @@ ActiveRecord::Schema.define(version: 20160331223143) do ...@@ -746,6 +747,8 @@ ActiveRecord::Schema.define(version: 20160331223143) do
t.boolean "public_builds", default: true, null: false t.boolean "public_builds", default: true, null: false
t.string "main_language" t.string "main_language"
t.integer "pushes_since_gc", default: 0 t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
end end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
...@@ -755,6 +758,7 @@ ActiveRecord::Schema.define(version: 20160331223143) do ...@@ -755,6 +758,7 @@ ActiveRecord::Schema.define(version: 20160331223143) do
add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree
add_index "projects", ["path"], name: "index_projects_on_path", using: :btree add_index "projects", ["path"], name: "index_projects_on_path", using: :btree
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
- [Environment Variables](administration/environment_variables.md) to configure GitLab. - [Environment Variables](administration/environment_variables.md) to configure GitLab.
- [Operations](operations/README.md) Keeping GitLab up and running - [Operations](operations/README.md) Keeping GitLab up and running
- [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects. - [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects.
- [Repository checks](administration/repository_checks.md) Periodic Git repository checks
- [Security](security/README.md) Learn what you can do to further secure your GitLab instance. - [Security](security/README.md) Learn what you can do to further secure your GitLab instance.
- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
- [Update](update/README.md) Update guides to upgrade your installation. - [Update](update/README.md) Update guides to upgrade your installation.
......
# Repository checks
>**Note:**
This feature was [introduced][ce-3232] in GitLab 8.7.
Git has a built-in mechanism, [git fsck][git-fsck], to verify the
integrity of all data commited to a repository. GitLab administrators
can trigger such a check for a project via the project page under the
admin panel. The checks run asynchronously so it may take a few minutes
before the check result is visible on the project admin page. If the
checks failed you can see their output on the admin log page under
'repocheck.log'.
## Periodic checks
GitLab periodically runs a repository check on all project repositories and
wiki repositories in order to detect data corruption problems. A
project will be checked no more than once per week. If any projects
fail their repository checks all GitLab administrators will receive an email
notification of the situation. This notification is sent out no more
than once a day.
## Disabling periodic checks
You can disable the periodic checks on the 'Settings' page of the admin
panel.
## What to do if a check failed
If the repository check fails for some repository you should look up the error
in repocheck.log (in the admin panel or on disk; see
`/var/log/gitlab/gitlab-rails` for Omnibus installations or
`/home/git/gitlab/log` for installations from source). Once you have
resolved the issue use the admin panel to trigger a new repository check on
the project. This will clear the 'check failed' state.
If for some reason the periodic repository check caused a lot of false
alarms you can choose to clear ALL repository check states from the
'Settings' page of the admin panel.
---
[ce-3232]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232 "Auto git fsck"
[git-fsck]: https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html "git fsck documentation"
\ No newline at end of file
...@@ -19,7 +19,7 @@ many projects, you can have a single or a small number of runners that handle ...@@ -19,7 +19,7 @@ many projects, you can have a single or a small number of runners that handle
multiple projects. This makes it easier to maintain and update runners. multiple projects. This makes it easier to maintain and update runners.
**Specific runners** are useful for jobs that have special requirements or for **Specific runners** are useful for jobs that have special requirements or for
projects with a very demand. If a job has certain requirements, you can set projects with a specific demand. If a job has certain requirements, you can set
up the specific runner with this in mind, while not having to do this for all up the specific runner with this in mind, while not having to do this for all
runners. For example, if you want to deploy a certain project, you can setup runners. For example, if you want to deploy a certain project, you can setup
a specific runner to have the right credentials for this. a specific runner to have the right credentials for this.
......
...@@ -20,11 +20,11 @@ class Spinach::Features::ProjectIssues < Spinach::FeatureSteps ...@@ -20,11 +20,11 @@ class Spinach::Features::ProjectIssues < Spinach::FeatureSteps
end end
step 'I should see that I am subscribed' do step 'I should see that I am subscribed' do
expect(find('.subscribe-button span')).to have_content 'Unsubscribe' expect(find('.issuable-subscribe-button span')).to have_content 'Unsubscribe'
end end
step 'I should see that I am unsubscribed' do step 'I should see that I am unsubscribed' do
expect(find('.subscribe-button span')).to have_content 'Subscribe' expect(find('.issuable-subscribe-button span')).to have_content 'Subscribe'
end end
step 'I click link "Closed"' do step 'I click link "Closed"' do
......
...@@ -29,6 +29,6 @@ class Spinach::Features::Labels < Spinach::FeatureSteps ...@@ -29,6 +29,6 @@ class Spinach::Features::Labels < Spinach::FeatureSteps
private private
def subscribe_button def subscribe_button
first('.subscribe-button span') first('.label-subscribe-button span')
end end
end end
...@@ -77,11 +77,11 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps ...@@ -77,11 +77,11 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps
end end
step 'I should see that I am subscribed' do step 'I should see that I am subscribed' do
expect(find('.subscribe-button span')).to have_content 'Unsubscribe' expect(find('.issuable-subscribe-button span')).to have_content 'Unsubscribe'
end end
step 'I should see that I am unsubscribed' do step 'I should see that I am unsubscribed' do
expect(find('.subscribe-button span')).to have_content 'Subscribe' expect(find('.issuable-subscribe-button span')).to have_content 'Subscribe'
end end
step 'I click button "Unsubscribe"' do step 'I click button "Unsubscribe"' do
......
...@@ -227,7 +227,7 @@ module SharedDiffNote ...@@ -227,7 +227,7 @@ module SharedDiffNote
end end
def click_diff_line(code) def click_diff_line(code)
find("button[data-line-code='#{code}']").click find("button[data-line-code='#{code}']").trigger('click')
end end
def click_parallel_diff_line(code, line_type) def click_parallel_diff_line(code, line_type)
......
...@@ -14,19 +14,29 @@ class AwardEmoji ...@@ -14,19 +14,29 @@ class AwardEmoji
food_drink: "Food" food_drink: "Food"
}.with_indifferent_access }.with_indifferent_access
CATEGORY_ALIASES = {
symbols: "objects_symbols",
foods: "food_drink",
travel: "travel_places"
}.with_indifferent_access
def self.normilize_emoji_name(name) def self.normilize_emoji_name(name)
aliases[name] || name aliases[name] || name
end end
def self.emoji_by_category def self.emoji_by_category
unless @emoji_by_category unless @emoji_by_category
@emoji_by_category = {} @emoji_by_category = Hash.new { |h, key| h[key] = [] }
emojis.each do |emoji_name, data| emojis.each do |emoji_name, data|
data["name"] = emoji_name data["name"] = emoji_name
@emoji_by_category[data["category"]] ||= [] # Skip Fitzpatrick(tone) modifiers
@emoji_by_category[data["category"]] << data next if data["category"] == "modifier"
category = CATEGORY_ALIASES[data["category"]] || data["category"]
@emoji_by_category[category] << data
end end
@emoji_by_category = @emoji_by_category.sort.to_h @emoji_by_category = @emoji_by_category.sort.to_h
......
...@@ -34,7 +34,8 @@ module Gitlab ...@@ -34,7 +34,8 @@ module Gitlab
max_artifacts_size: Settings.artifacts['max_size'], max_artifacts_size: Settings.artifacts['max_size'],
require_two_factor_authentication: false, require_two_factor_authentication: false,
two_factor_grace_period: 48, two_factor_grace_period: 48,
akismet_enabled: false akismet_enabled: false,
repository_checks_enabled: true,
) )
end end
......
module Gitlab
class RepositoryCheckLogger < Gitlab::Logger
def self.file_name_noext
'repocheck'
end
end
end
# GITLAB CI
server {
listen 80 default_server; # e.g., listen 192.168.1.1:80;
server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com;
access_log /var/log/nginx/gitlab_ci_access.log;
error_log /var/log/nginx/gitlab_ci_error.log;
# expose API to fix runners
location /api {
proxy_read_timeout 300;
proxy_connect_timeout 300;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
# You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN
resolver 8.8.8.8 8.8.4.4;
proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri;
}
# redirect all other CI requests
location / {
return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri;
}
# adjust this to match the largest build log your runners might submit,
# set to 0 to disable limit
client_max_body_size 10m;
}
\ No newline at end of file
require 'rails_helper'
feature 'Admin uses repository checks', feature: true do
before { login_as :admin }
scenario 'to trigger a single check' do
project = create(:empty_project)
visit_admin_project_page(project)
page.within('.repository-check') do
click_button 'Trigger repository check'
end
expect(page).to have_content('Repository check was triggered')
end
scenario 'to see a single failed repository check' do
project = create(:empty_project)
project.update_columns(
last_repository_check_failed: true,
last_repository_check_at: Time.now,
)
visit_admin_project_page(project)
page.within('.alert') do
expect(page.text).to match(/Last repository check \(.* ago\) failed/)
end
end
scenario 'to clear all repository checks', js: true do
visit admin_application_settings_path
expect(RepositoryCheck::ClearWorker).to receive(:perform_async)
click_link 'Clear all repository checks'
expect(page).to have_content('Started asynchronous removal of all repository check states.')
end
def visit_admin_project_page(project)
visit admin_namespace_project_path(project.namespace, project)
end
end
...@@ -16,4 +16,11 @@ describe AwardEmoji do ...@@ -16,4 +16,11 @@ describe AwardEmoji do
end end
end end
end end
describe '.emoji_by_category' do
it "only contains known categories" do
undefined_categories = AwardEmoji.emoji_by_category.keys - AwardEmoji::CATEGORIES.keys
expect(undefined_categories).to be_empty
end
end
end end
require 'rails_helper'
describe RepositoryCheckMailer do
include EmailSpec::Matchers
describe '.notify' do
it 'emails all admins' do
admins = create_list(:admin, 3)
mail = described_class.notify(1)
expect(mail).to deliver_to admins.map(&:email)
end
it 'mentions the number of failed checks' do
mail = described_class.notify(3)
expect(mail).to have_subject '3 projects failed their last repository check'
end
end
end
...@@ -94,6 +94,12 @@ describe Repository, models: true do ...@@ -94,6 +94,12 @@ describe Repository, models: true do
it { is_expected.to be_an Array } it { is_expected.to be_an Array }
it 'regex-escapes the query string' do
results = repository.search_files("test\\", 'master')
expect(results.first).not_to start_with('fatal:')
end
describe 'result' do describe 'result' do
subject { results.first } subject { results.first }
......
require 'spec_helper'
describe RepositoryCheck::BatchWorker do
subject { described_class.new }
it 'prefers projects that have never been checked' do
projects = create_list(:project, 3)
projects[0].update_column(:last_repository_check_at, 4.months.ago)
projects[2].update_column(:last_repository_check_at, 3.months.ago)
expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
end
it 'sorts projects by last_repository_check_at' do
projects = create_list(:project, 3)
projects[0].update_column(:last_repository_check_at, 2.months.ago)
projects[1].update_column(:last_repository_check_at, 4.months.ago)
projects[2].update_column(:last_repository_check_at, 3.months.ago)
expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
end
it 'excludes projects that were checked recently' do
projects = create_list(:project, 3)
projects[0].update_column(:last_repository_check_at, 2.days.ago)
projects[1].update_column(:last_repository_check_at, 2.months.ago)
projects[2].update_column(:last_repository_check_at, 3.days.ago)
expect(subject.perform).to eq([projects[1].id])
end
it 'does nothing when repository checks are disabled' do
create(:empty_project)
current_settings = double('settings', repository_checks_enabled: false)
expect(subject).to receive(:current_settings) { current_settings }
expect(subject.perform).to eq(nil)
end
end
require 'spec_helper'
describe RepositoryCheck::ClearWorker do
it 'clears repository check columns' do
project = create(:empty_project)
project.update_columns(
last_repository_check_failed: true,
last_repository_check_at: Time.now,
)
described_class.new.perform
project.reload
expect(project.last_repository_check_failed).to be_nil
expect(project.last_repository_check_at).to be_nil
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment