Merge branch '19376-post-bfg-cleanup' into 'master'

Allow internal references to be removed

Closes #19376

See merge request gitlab-org/gitlab-ce!23189

Gemfile

@@ -432,7 +432,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 1.2.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 1.3.0', require: 'gitaly'
 gem 'grpc', '~> 1.15.0'
 
 gem 'google-protobuf', '~> 3.6'

Gemfile.lock

@@ -274,7 +274,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.2.0)
+    gitaly-proto (1.3.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-default_value_for (3.1.1)
@@ -1007,7 +1007,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.2.0)
+  gitaly-proto (~> 1.3.0)
   github-markup (~> 1.7.0)
   gitlab-default_value_for (~> 3.1.1)
   gitlab-markup (~> 1.6.5)

Gemfile.rails4.lock

@@ -273,7 +273,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (1.2.0)
+    gitaly-proto (1.3.0)
       grpc (~> 1.0)
     github-markup (1.7.0)
     gitlab-markup (1.6.5)
@@ -999,7 +999,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 1.2.0)
+  gitaly-proto (~> 1.3.0)
   github-markup (~> 1.7.0)
   gitlab-markup (~> 1.6.5)
   gitlab-sidekiq-fetcher

app/assets/javascripts/behaviors/requires_input.js

@@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) {
 }
 
 $(() => {
-  const $form = $('form.js-requires-input');
-  if ($form) {
+  $('form.js-requires-input').each((i, el) => {
+    const $form = $(el);
+
     $form.requiresInput();
     hideOrShowHelpBlock($form);
     $('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form));
-  }
+  });
 });

app/assets/javascripts/lib/utils/file_upload.js

+export default (buttonSelector, fileSelector) => {
+  const btn = document.querySelector(buttonSelector);
+  const fileInput = document.querySelector(fileSelector);
+  const form = btn.closest('form');
+
+  btn.addEventListener('click', () => {
+    fileInput.click();
+  });
+
+  fileInput.addEventListener('change', () => {
+    form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
+  });
+};

app/assets/javascripts/pages/projects/edit/index.js

@@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels';
 import setupProjectEdit from '~/project_edit';
 import initConfirmDangerModal from '~/confirm_danger_modal';
 import mountBadgeSettings from '~/pages/shared/mount_badge_settings';
+import fileUpload from '~/lib/utils/file_upload';
 import initProjectLoadingSpinner from '../shared/save_project_loader';
-import projectAvatar from '../shared/project_avatar';
 import initProjectPermissionsSettings from '../shared/permissions';
 
 document.addEventListener('DOMContentLoaded', () => {
@@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => {
   setupProjectEdit();
   // Initialize expandable settings panels
   initSettingsPanels();
-  projectAvatar();
+  fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input');
   initProjectPermissionsSettings();
   initConfirmDangerModal();
   mountBadgeSettings(PROJECT_BADGE);

app/assets/javascripts/pages/projects/settings/repository/form.js

@@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys';
 import ProtectedBranchCreate from '~/protected_branches/protected_branch_create';
 import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list';
 import DueDateSelectors from '~/due_date_select';
+import fileUpload from '~/lib/utils/file_upload';
 
 export default () => {
   new ProtectedTagCreate();
@@ -16,4 +17,5 @@ export default () => {
   new ProtectedBranchCreate();
   new ProtectedBranchEditList();
   new DueDateSelectors();
+  fileUpload('.js-choose-file', '.js-object-map-input');
 };

app/assets/javascripts/pages/projects/shared/project_avatar.js

-import $ from 'jquery';
-
-export default function projectAvatar() {
-  $('.js-choose-project-avatar-button').bind('click', function onClickAvatar() {
-    const form = $(this).closest('form');
-    return form.find('.js-project-avatar-input').click();
-  });
-
-  $('.js-project-avatar-input').bind('change', function onClickAvatarInput() {
-    const form = $(this).closest('form');
-    const filename = $(this)
-      .val()
-      .replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
-    return form.find('.js-avatar-filename').text(filename);
-  });
-}

app/controllers/projects/settings/repository_controller.rb

@@ -5,6 +5,7 @@ module Projects
     class RepositoryController < Projects::ApplicationController
       before_action :authorize_admin_project!
       before_action :remote_mirror, only: [:show]
+      before_action :check_cleanup_feature_flag!, only: :cleanup
 
       def show
         render_show
@@ -20,8 +21,26 @@ module Projects
         render_show
       end
 
+      def cleanup
+        cleanup_params = params.require(:project).permit(:bfg_object_map)
+        result = Projects::UpdateService.new(project, current_user, cleanup_params).execute
+
+        if result[:status] == :success
+          RepositoryCleanupWorker.perform_async(project.id, current_user.id)
+          flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.')
+        else
+          flash[:alert] = _('Failed to upload object map file')
+        end
+
+        redirect_to project_settings_repository_path(project)
+      end
+
       private
 
+      def check_cleanup_feature_flag!
+        render_404 unless ::Feature.enabled?(:project_cleanup, project)
+      end
+
       def render_show
         @deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user)
         @deploy_tokens = @project.deploy_tokens.active

app/helpers/projects_helper.rb

@@ -257,6 +257,10 @@ module ProjectsHelper
     "xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
   end
 
+  def link_to_bfg
+    link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer'
+  end
+
   def legacy_render_context(params)
     params[:legacy_render] ? { markdown_engine: :redcarpet } : {}
   end

app/mailers/emails/projects.rb

@@ -24,6 +24,21 @@ module Emails
            subject: subject("Project export error"))
     end
 
+    def repository_cleanup_success_email(project, user)
+      @project = project
+      @user = user
+
+      mail(to: user.notification_email, subject: subject("Project cleanup has completed"))
+    end
+
+    def repository_cleanup_failure_email(project, user, error)
+      @project = project
+      @user = user
+      @error = error
+
+      mail(to: user.notification_email, subject: subject("Project cleanup failure"))
+    end
+
     def repository_push_email(project_id, opts = {})
       @message =
         Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)

app/models/project.rb

@@ -339,6 +339,7 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
   validates :variables, variable_duplicates: { scope: :environment_scope }
+  validates :bfg_object_map, file_size: { maximum: :max_attachment_size }
 
   # Scopes
   scope :pending_delete, -> { where(pending_delete: true) }
@@ -412,6 +413,9 @@ class Project < ActiveRecord::Base
                                             only_integer: true,
                                             message: 'needs to be beetween 10 minutes and 1 month' }
 
+  # Used by Projects::CleanupService to hold a map of rewritten object IDs
+  mount_uploader :bfg_object_map, AttachmentUploader
+
   # Returns a project, if it is not about to be removed.
   #
   # id - The ID of the project to retrieve.
@@ -1973,6 +1977,10 @@ class Project < ActiveRecord::Base
     Ability.allowed?(user, :read_project_snippet, self)
   end
 
+  def max_attachment_size
+    Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i
+  end
+
   private
 
   def use_hashed_storage

app/services/notification_service.rb

@@ -466,6 +466,14 @@ class NotificationService
     end
   end
 
+  def repository_cleanup_success(project, user)
+    mailer.send(:repository_cleanup_success_email, project, user).deliver_later
+  end
+
+  def repository_cleanup_failure(project, user, error)
+    mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later
+  end
+
   protected
 
   def new_resource_email(target, method)

app/services/projects/cleanup_service.rb

+# frozen_string_literal: true
+
+module Projects
+  # The CleanupService removes data from the project repository following a
+  # BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
+  #
+  # Before executing this service, all refs rewritten by BFG should have been
+  # pushed to the repository
+  class CleanupService < BaseService
+    NoUploadError = StandardError.new("Couldn't find uploaded object map")
+
+    include Gitlab::Utils::StrongMemoize
+
+    # Attempt to clean up the project following the push. Warning: this is
+    # destructive!
+    #
+    # path is the path of an upload of a BFG object map file. It contains a line
+    # per rewritten object, with the old and new SHAs space-separated. It can be
+    # used to update or remove content that references the objects that BFG has
+    # altered
+    #
+    # Currently, only the project repository is modified by this service, but we
+    # may wish to modify other data sources in the future.
+    def execute
+      apply_bfg_object_map!
+
+      # Remove older objects that are no longer referenced
+      GitGarbageCollectWorker.new.perform(project.id, :gc)
+
+      # The cache may now be inaccurate, and holding onto it could prevent
+      # bugs assuming the presence of some object from manifesting for some
+      # time. Better to feel the pain immediately.
+      project.repository.expire_all_method_caches
+
+      project.bfg_object_map.remove!
+    end
+
+    private
+
+    def apply_bfg_object_map!
+      raise NoUploadError unless project.bfg_object_map.exists?
+
+      project.bfg_object_map.open do |io|
+        repository_cleaner.apply_bfg_object_map(io)
+      end
+    end
+
+    def repository_cleaner
+      @repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw)
+    end
+  end
+end

app/views/errors/access_denied.html.haml

@@ -9,7 +9,7 @@
     %p
       = message
   %p
-    = s_('403|Please contact your GitLab administrator to get the permission.')
+    = s_('403|Please contact your GitLab administrator to get permission.')
   .action-container.js-go-back{ style: 'display: none' }
     %a{ href: 'javascript:history.back()', class: 'btn btn-success' }
       = s_('Go Back')

app/views/notify/repository_cleanup_failure_email.text.erb

+Repository cleanup failed on <%= @project.web_url %>
+
+<%= @error %>

app/views/notify/repository_cleanup_success_email.text.erb

+Repository cleanup succeeded on <%= @project.web_url %>
+
+Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB

app/views/projects/cleanup/_show.html.haml

+- return unless Feature.enabled?(:project_cleanup, @project)
+
+- expanded = Rails.env.test?
+
+%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4= _('Repository cleanup')
+    %button.btn.js-settings-toggle
+      = expanded ? _('Collapse') : _('Expand')
+    %p
+      = _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe
+      = link_to icon('question-circle'),
+          help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'),
+          target: '_blank', rel: 'noopener noreferrer'
+
+  .settings-content
+    - url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project)
+    = form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f|
+      %fieldset.prepend-top-0.append-bottom-10
+        .append-bottom-10
+          %h5.prepend-top-0
+            = _("Upload object map")
+          %button.btn.btn-default.js-choose-file{ type: "button" }
+            = _("Choose a file")
+          %span.prepend-left-default.js-filename
+            = _("No file selected")
+          = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
+        .form-text.text-muted
+          = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
+      = f.submit _('Start cleanup'), class: 'btn btn-success'
+

app/views/projects/edit.html.haml

@@ -53,7 +53,7 @@
                 = _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git }
             .prepend-top-5.append-bottom-10
               %button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...")
-              %span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen")
+              %span.file_name.prepend-left-default.js-filename= _("No file chosen")
               = f.file_field :avatar, class: "js-project-avatar-input hidden"
             .form-text.text-muted= _("The maximum file size allowed is 200KB.")
             - if @project.avatar?

app/views/projects/settings/repository/show.html.haml

@@ -13,3 +13,4 @@
 = render "projects/protected_tags/index"
 = render @deploy_keys
 = render "projects/deploy_tokens/index"
+= render "projects/cleanup/show"

app/workers/all_queues.yml

@@ -133,3 +133,4 @@
 - create_note_diff_file
 - delete_diff_files
 - detect_repository_languages
+- repository_cleanup

app/workers/repository_cleanup_worker.rb

+# frozen_string_literal: true
+
+class RepositoryCleanupWorker
+  include ApplicationWorker
+
+  sidekiq_options retry: 3
+
+  sidekiq_retries_exhausted do |msg, err|
+    next if err.is_a?(ActiveRecord::RecordNotFound)
+
+    args = msg['args'] + [msg['error_message']]
+
+    new.perform_failure(*args)
+  end
+
+  def perform(project_id, user_id)
+    project = Project.find(project_id)
+    user = User.find(user_id)
+
+    Projects::CleanupService.new(project, user).execute
+
+    notification_service.repository_cleanup_success(project, user)
+  end
+
+  def perform_failure(project_id, user_id, error)
+    project = Project.find(project_id)
+    user = User.find(user_id)
+
+    # Ensure the file is removed
+    project.bfg_object_map.remove!
+    notification_service.repository_cleanup_failure(project, user, error)
+  end
+
+  private
+
+  def notification_service
+    @notification_service ||= NotificationService.new
+  end
+end

changelogs/unreleased/19376-post-bfg-cleanup.yml

+---
+title: Use BFG object maps to clean projects
+merge_request: 23189
+author:
+type: added

config/routes/project.rb

@@ -436,6 +436,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         resource :integrations, only: [:show]
         resource :repository, only: [:show], controller: :repository do
           post :create_deploy_token, path: 'deploy_token/create'
+          post :cleanup
         end
       end
 

config/sidekiq_queues.yml

@@ -81,3 +81,4 @@
   - [delete_diff_files, 1]
   - [detect_repository_languages, 1]
   - [auto_devops, 2]
+  - [repository_cleanup, 1]

db/migrate/20181203002526_add_project_bfg_object_map_column.rb

+# frozen_string_literal: true
+
+class AddProjectBfgObjectMapColumn < ActiveRecord::Migration[5.0]
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :bfg_object_map, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20181129104944) do
+ActiveRecord::Schema.define(version: 20181203002526) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1684,6 +1684,7 @@ ActiveRecord::Schema.define(version: 20181129104944) do
     t.boolean "remote_mirror_available_overridden"
     t.bigint "pool_repository_id"
     t.string "runners_token_encrypted"
+    t.string "bfg_object_map"
     t.index ["ci_id"], name: "index_projects_on_ci_id", using: :btree
     t.index ["created_at"], name: "index_projects_on_created_at", using: :btree
     t.index ["creator_id"], name: "index_projects_on_creator_id", using: :btree

doc/user/project/repository/reducing_the_repo_size_using_git.md

 # Reducing the repository size using Git
 
 A GitLab Enterprise Edition administrator can set a [repository size limit][admin-repo-size]
-which will prevent you to exceed it.
+which will prevent you from exceeding it.
 
 When a project has reached its size limit, you will not be able to push to it,
 create a new merge request, or merge existing ones. You will still be able to
 create new issues, and clone the project though. Uploading LFS objects will
 also be denied.
 
-In order to lift these restrictions, the administrator of the GitLab instance
-needs to increase the limit on the particular project that exceeded it or you
-need to instruct Git to rewrite changes.
-
 If you exceed the repository size limit, your first thought might be to remove
-some data, make a new commit and push back to the repository. Unfortunately,
-it's not so easy and that workflow won't work. Deleting files in a commit doesn't
-actually reduce the size of the repo since the earlier commits and blobs are
-still around. What you need to do is rewrite history with Git's
-[`filter-branch` option][gitscm].
+some data, make a new commit and push back to the repository. Perhaps you can
+move some blobs to LFS, or remove some old dependency updates from history.
+Unfortunately, it's not so easy and that workflow won't work. Deleting files in
+a commit doesn't actually reduce the size of the repo since the earlier commits
+and blobs are still around. What you need to do is rewrite history with Git's
+[`filter-branch` option][gitscm], or a tool like the [BFG Repo-Cleaner][bfg].
 
 Note that even with that method, until `git gc` runs on the GitLab side, the
-"removed" commits and blobs will still be around. And if a commit was ever
-included in an MR, or if a build was run for a commit, or if a user commented
-on it, it will be kept around too. So, in these cases the size will not decrease.
-
-The only fool proof way to actually decrease the repository size is to prune all
-the unneeded stuff locally, and then create a new project on GitLab and start
-using that instead.
+"removed" commits and blobs will still be around. You also need to be able to
+push the rewritten history to GitLab, which may be impossible if you've already
+exceeded the maximum size limit.
 
-With that being said, you can try reducing your repository size with the
-following method.
-
-## Using `git filter-branch` to purge files
+In order to lift these restrictions, the administrator of the GitLab instance
+needs to increase the limit on the particular project that exceeded it, so it's
+always better to spot that you're approaching the limit and act proactively to
+stay underneath it. If you hit the limit, and your admin can't - or won't -
+temporarily increase it for you, your only option is to prune all the unneeded
+stuff locally, and then create a new project on GitLab and start using that
+instead.
+
+If you can continue to use the original project, we recommend [using the
+BFG Repo-Cleaner](#using-the-bfg-repo-cleaner). It's faster and simpler than
+`git filter-branch`, and GitLab can use its account of what has changed to clean
+up its own internal state, maximizing the space saved.
 
 > **Warning:**
 > Make sure to first make a copy of your repository since rewriting history will
 > purge the files and information you are about to delete. Also make sure to
 > inform any collaborators to not use `pull` after your changes, but use `rebase`.
 
+> **Warning:**
+> This process is not suitable for removing sensitive data like password or keys
+> from your repository. Information about commits, including file content, is
+> cached in the database, and will remain visible even after they have been
+> removed from the repository.
+
+## Using the BFG Repo-Cleaner
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/19376) in GitLab 11.6.
+
+1. [Install BFG](https://rtyley.github.io/bfg-repo-cleaner/).
+
+1. Navigate to your repository:
+
+    ```
+    cd my_repository/
+    ```
+
+1. Change to the branch you want to remove the big file from:
+
+    ```
+    git checkout master
+    ```
+
+1. Create a commit removing the large file from the branch, if it still exists:
+
+    ```
+    git rm path/to/big_file.mpg
+    git commit -m 'Remove unneeded large file'
+    ```
+
+1. Rewrite history:
+
+    ```
+    bfg --delete-files path/to/big_file.mpg
+    ```
+
+    An object map file will be written to `object-id-map.old-new.txt`. Keep it
+    around - you'll need it for the final step!
+
+1. Force-push the changes to GitLab:
+
+    ```
+    git push --force-with-lease origin master
+    ```
+
+    If this step fails, someone has changed the `master` branch while you were
+    rewriting history. You could restore the branch and re-run BFG to preserve
+    their changes, or use `git push --force` to overwrite their changes.
+
+1. Navigate to **Project > Settings > Repository > Repository Cleanup**:
+
+    ![Repository settings cleanup form](img/repository_cleanup.png)
+
+    Upload the `object-id-map.old-new.txt` file and press **Start cleanup**.
+    This will remove any internal git references to the old commits, and run
+    `git gc` against the repository. You will receive an email once it has
+    completed.
+
+## Using `git filter-branch`
+
 1. Navigate to your repository:
 
     ```
@@ -70,11 +132,6 @@ following method.
 
 Your repository should now be below the size limit.
 
-> **Note:**
-> As an alternative to `filter-branch`, you can use the `bfg` tool with a
-> command like: `bfg --delete-files path/to/big_file.mpg`. Read the
-> [BFG Repo-Cleaner][bfg] documentation for more information.
-
 [admin-repo-size]: https://docs.gitlab.com/ee/user/admin_area/settings/account_and_limit_settings.html#repository-size-limit
 [bfg]: https://rtyley.github.io/bfg-repo-cleaner/
 [gitscm]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#The-Nuclear-Option:-filter-branch

lib/gitlab/git/repository_cleaner.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Git
+    class RepositoryCleaner
+      include Gitlab::Git::WrapsGitalyErrors
+
+      attr_reader :repository
+
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @repository = repository
+      end
+
+      def apply_bfg_object_map(io)
+        wrapped_gitaly_errors do
+          gitaly_cleanup_client.apply_bfg_object_map(io)
+        end
+      end
+
+      private
+
+      def gitaly_cleanup_client
+        @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository)
+      end
+    end
+  end
+end

lib/gitlab/gitaly_client/cleanup_service.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module GitalyClient
+    class CleanupService
+      attr_reader :repository, :gitaly_repo, :storage
+
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @repository = repository
+        @gitaly_repo = repository.gitaly_repository
+        @storage = repository.storage
+      end
+
+      def apply_bfg_object_map(io)
+        first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo)
+
+        enum = Enumerator.new do |y|
+          y.yield first_request
+
+          while data = io.read(RepositoryService::MAX_MSG_SIZE)
+            y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data)
+          end
+        end
+
+        GitalyClient.call(
+          storage,
+          :cleanup_service,
+          :apply_bfg_object_map,
+          enum,
+          timeout: GitalyClient.no_timeout
+        )
+      end
+    end
+  end
+end

lib/gitlab/import_export/import_export.yml

@@ -115,6 +115,7 @@ excluded_attributes:
     - :remote_mirror_available_overridden
     - :description_html
     - :repository_languages
+    - :bfg_object_map
   namespaces:
     - :runners_token
     - :runners_token_encrypted

locale/gitlab.pot

@@ -225,7 +225,7 @@ msgstr ""
 msgid "2FA enabled"
 msgstr ""
 
-msgid "403|Please contact your GitLab administrator to get the permission."
+msgid "403|Please contact your GitLab administrator to get permission."
 msgstr ""
 
 msgid "403|You don't have the permission to access this page."
@@ -1244,6 +1244,9 @@ msgstr ""
 msgid "Choose a branch/tag (e.g. %{master}) or enter a commit (e.g. %{sha}) to see what's changed or to create a merge request."
 msgstr ""
 
+msgid "Choose a file"
+msgstr ""
+
 msgid "Choose a template..."
 msgstr ""
 
@@ -2939,6 +2942,9 @@ msgstr ""
 msgid "Failed to update issues, please try again."
 msgstr ""
 
+msgid "Failed to upload object map file"
+msgstr ""
+
 msgid "Failure"
 msgstr ""
 
@@ -4333,6 +4339,9 @@ msgstr ""
 msgid "No file chosen"
 msgstr ""
 
+msgid "No file selected"
+msgstr ""
+
 msgid "No files found."
 msgstr ""
 
@@ -5489,6 +5498,12 @@ msgstr ""
 msgid "Repository URL"
 msgstr ""
 
+msgid "Repository cleanup"
+msgstr ""
+
+msgid "Repository cleanup has started. You will receive an email once the cleanup operation is complete."
+msgstr ""
+
 msgid "Repository maintenance"
 msgstr ""
 
@@ -6201,6 +6216,9 @@ msgstr ""
 msgid "Start and due date"
 msgstr ""
 
+msgid "Start cleanup"
+msgstr ""
+
 msgid "Start date"
 msgstr ""
 
@@ -6416,6 +6434,9 @@ msgstr ""
 msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage."
 msgstr ""
 
+msgid "The maximum file size allowed is %{max_attachment_size}mb"
+msgstr ""
+
 msgid "The maximum file size allowed is 200KB."
 msgstr ""
 
@@ -7098,6 +7119,9 @@ msgstr ""
 msgid "Upload file"
 msgstr ""
 
+msgid "Upload object map"
+msgstr ""
+
 msgid "UploadLink|click to upload"
 msgstr ""
 

spec/controllers/projects/settings/repository_controller_spec.rb

@@ -17,4 +17,35 @@ describe Projects::Settings::RepositoryController do
       expect(response).to render_template(:show)
     end
   end
+
+  describe 'PUT cleanup' do
+    def do_put!
+      object_map = fixture_file_upload('spec/fixtures/bfg_object_map.txt')
+
+      Sidekiq::Testing.fake! do
+        put :cleanup, namespace_id: project.namespace, project_id: project, project: { object_map: object_map }
+      end
+    end
+
+    context 'feature enabled' do
+      it 'enqueues a RepositoryCleanupWorker' do
+        stub_feature_flags(project_cleanup: true)
+
+        do_put!
+
+        expect(response).to redirect_to project_settings_repository_path(project)
+        expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+      end
+    end
+
+    context 'feature disabled' do
+      it 'shows a 404 error' do
+        stub_feature_flags(project_cleanup: false)
+
+        do_put!
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
 end

spec/features/projects/settings/repository_settings_spec.rb

@@ -196,5 +196,40 @@ describe 'Projects > Settings > Repository settings' do
         end
       end
     end
+
+    context 'repository cleanup settings' do
+      let(:object_map_file) { Rails.root.join('spec', 'fixtures', 'bfg_object_map.txt') }
+
+      context 'feature enabled' do
+        it 'uploads an object map file', :js do
+          stub_feature_flags(project_cleanup: true)
+
+          visit project_settings_repository_path(project)
+
+          expect(page).to have_content('Repository cleanup')
+
+          page.within('#cleanup') do
+            attach_file('project[bfg_object_map]', object_map_file, visible: false)
+
+            Sidekiq::Testing.fake! do
+              click_button 'Start cleanup'
+            end
+          end
+
+          expect(page).to have_content('Repository cleanup has started')
+          expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+        end
+      end
+
+      context 'feature disabled' do
+        it 'does not show the settings' do
+          stub_feature_flags(project_cleanup: false)
+
+          visit project_settings_repository_path(project)
+
+          expect(page).not_to have_content('Repository cleanup')
+        end
+      end
+    end
   end
 end

spec/fixtures/bfg_object_map.txt

+f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 e242ed3bffccdf271b7fbaf34ed72d089537b42f

spec/helpers/projects_helper_spec.rb

@@ -471,6 +471,31 @@ describe ProjectsHelper do
     end
   end
 
+  describe 'link_to_bfg' do
+    subject { helper.link_to_bfg }
+
+    it 'generates a hardcoded link to the BFG Repo-Cleaner' do
+      result = helper.link_to_bfg
+      doc = Nokogiri::HTML.fragment(result)
+
+      expect(doc.children.size).to eq(1)
+
+      link = doc.children.first
+
+      aggregate_failures do
+        expect(result).to be_html_safe
+
+        expect(link.name).to eq('a')
+        expect(link[:target]).to eq('_blank')
+        expect(link[:rel]).to eq('noopener noreferrer')
+        expect(link[:href]).to eq('https://rtyley.github.io/bfg-repo-cleaner/')
+        expect(link.inner_html).to eq('BFG')
+
+        expect(result).to be_html_safe
+      end
+    end
+  end
+
   describe '#legacy_render_context' do
     it 'returns the redcarpet engine' do
       params = { legacy_render: '1' }

spec/javascripts/lib/utils/file_upload_spec.js

+import fileUpload from '~/lib/utils/file_upload';
+
+describe('File upload', () => {
+  beforeEach(() => {
+    setFixtures(`
+      <form>
+        <button class="js-button" type="button">Click me!</button>
+        <input type="text" class="js-input" />
+        <span class="js-filename"></span>
+      </form>
+    `);
+
+    fileUpload('.js-button', '.js-input');
+  });
+
+  it('clicks file input after clicking button', () => {
+    const btn = document.querySelector('.js-button');
+    const input = document.querySelector('.js-input');
+
+    spyOn(input, 'click');
+
+    btn.click();
+
+    expect(input.click).toHaveBeenCalled();
+  });
+
+  it('updates file name text', () => {
+    const input = document.querySelector('.js-input');
+
+    input.value = 'path/to/file/index.js';
+
+    input.dispatchEvent(new CustomEvent('change'));
+
+    expect(document.querySelector('.js-filename').textContent).toEqual('index.js');
+  });
+});

spec/lib/gitlab/git/repository_cleaner_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Git::RepositoryCleaner do
+  let(:project) { create(:project, :repository) }
+  let(:repository) { project.repository }
+  let(:head_sha) { repository.head_commit.id }
+
+  let(:object_map) { StringIO.new("#{head_sha} #{'0' * 40}") }
+
+  subject(:cleaner) { described_class.new(repository.raw) }
+
+  describe '#apply_bfg_object_map' do
+    it 'removes internal references pointing at SHAs in the object map' do
+      # Create some refs we expect to be removed
+      repository.keep_around(head_sha)
+      repository.create_ref(head_sha, 'refs/environments/1')
+      repository.create_ref(head_sha, 'refs/merge-requests/1')
+      repository.create_ref(head_sha, 'refs/heads/_keep')
+      repository.create_ref(head_sha, 'refs/tags/_keep')
+
+      cleaner.apply_bfg_object_map(object_map)
+
+      aggregate_failures do
+        expect(repository.kept_around?(head_sha)).to be_falsy
+        expect(repository.ref_exists?('refs/environments/1')).to be_falsy
+        expect(repository.ref_exists?('refs/merge-requests/1')).to be_falsy
+        expect(repository.ref_exists?('refs/heads/_keep')).to be_truthy
+        expect(repository.ref_exists?('refs/tags/_keep')).to be_truthy
+      end
+    end
+  end
+end

spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb

+require 'spec_helper'
+
+describe Gitlab::GitalyClient::CleanupService do
+  let(:project) { create(:project) }
+  let(:storage_name) { project.repository_storage }
+  let(:relative_path) { project.disk_path + '.git' }
+  let(:client) { described_class.new(project.repository) }
+
+  describe '#apply_bfg_object_map' do
+    it 'sends an apply_bfg_object_map message' do
+      expect_any_instance_of(Gitaly::CleanupService::Stub)
+        .to receive(:apply_bfg_object_map)
+        .with(kind_of(Enumerator), kind_of(Hash))
+        .and_return(double)
+
+      client.apply_bfg_object_map(StringIO.new)
+    end
+  end
+end

spec/services/notification_service_spec.rb

@@ -2146,6 +2146,27 @@ describe NotificationService, :mailer do
     end
   end
 
+  describe 'Repository cleanup' do
+    let(:user) { create(:user) }
+    let(:project) { create(:project) }
+
+    describe '#repository_cleanup_success' do
+      it 'emails the specified user only' do
+        notification.repository_cleanup_success(project, user)
+
+        should_email(user)
+      end
+    end
+
+    describe '#repository_cleanup_failure' do
+      it 'emails the specified user only' do
+        notification.repository_cleanup_failure(project, user, 'Some error')
+
+        should_email(user)
+      end
+    end
+  end
+
   def build_team(project)
     @u_watcher               = create_global_setting_for(create(:user), :watch)
     @u_participating         = create_global_setting_for(create(:user), :participating)

spec/services/projects/cleanup_service_spec.rb

+require 'spec_helper'
+
+describe Projects::CleanupService do
+  let(:project) { create(:project, :repository, bfg_object_map: fixture_file_upload('spec/fixtures/bfg_object_map.txt')) }
+  let(:object_map) { project.bfg_object_map }
+
+  subject(:service) { described_class.new(project) }
+
+  describe '#execute' do
+    it 'runs the apply_bfg_object_map gitaly RPC' do
+      expect_next_instance_of(Gitlab::Git::RepositoryCleaner) do |cleaner|
+        expect(cleaner).to receive(:apply_bfg_object_map).with(kind_of(IO))
+      end
+
+      service.execute
+    end
+
+    it 'runs garbage collection on the repository' do
+      expect_next_instance_of(GitGarbageCollectWorker) do |worker|
+        expect(worker).to receive(:perform)
+      end
+
+      service.execute
+    end
+
+    it 'clears the repository cache' do
+      expect(project.repository).to receive(:expire_all_method_caches)
+
+      service.execute
+    end
+
+    it 'removes the object map file' do
+      service.execute
+
+      expect(object_map.exists?).to be_falsy
+    end
+
+    it 'raises an error if no object map can be found' do
+      object_map.remove!
+
+      expect { service.execute }.to raise_error(described_class::NoUploadError)
+    end
+  end
+end

spec/workers/repository_cleanup_worker_spec.rb

+require 'spec_helper'
+
+describe RepositoryCleanupWorker do
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    it 'executes the cleanup service and sends a success notification' do
+      expect_next_instance_of(Projects::CleanupService) do |service|
+        expect(service.project).to eq(project)
+        expect(service.current_user).to eq(user)
+
+        expect(service).to receive(:execute)
+      end
+
+      expect_next_instance_of(NotificationService) do |service|
+        expect(service).to receive(:repository_cleanup_success).with(project, user)
+      end
+
+      worker.perform(project.id, user.id)
+    end
+
+    it 'raises an error if the project cannot be found' do
+      project.destroy
+
+      expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+
+    it 'raises an error if the user cannot be found' do
+      user.destroy
+
+      expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+
+  describe '#sidekiq_retries_exhausted' do
+    let(:job) { { 'args' => [project.id, user.id], 'error_message' => 'Error' } }
+
+    it 'does not send a failure notification for a RecordNotFound error' do
+      expect(NotificationService).not_to receive(:new)
+
+      described_class.sidekiq_retries_exhausted_block.call(job, ActiveRecord::RecordNotFound.new)
+    end
+
+    it 'sends a failure notification' do
+      expect_next_instance_of(NotificationService) do |service|
+        expect(service).to receive(:repository_cleanup_failure).with(project, user, 'Error')
+      end
+
+      described_class.sidekiq_retries_exhausted_block.call(job, StandardError.new)
+    end
+  end
+end