Squashed commit of the following:

commit ad59f123
Author: Lin Jen-Shin <godfat@godfat.org>
Date:   Sat Feb 4 00:05:31 2017 +0800

    Test both execute and async_execute

commit 1868b8af
Author: Lin Jen-Shin <godfat@godfat.org>
Date:   Fri Feb 3 23:44:35 2017 +0800

    Move the tests to spec/services/projects/destroy_service_spec.rb

commit aaf382d5
Author: Lin Jen-Shin <godfat@godfat.org>
Date:   Fri Feb 3 23:21:10 2017 +0800

    Skip or retain project while deleting the project:

    * Skip Ci::Build#update_project_statistics whenever there's no
      project (i.e. we're deleting the project!)
    * Retain the unscoped_project before deleting the build,
      so that we could use the data to delete the artifacts.
      Note that carrierwave uses `after_commit` for this,
      so we need to retain it in the memory.

    Closes #15005

app/models/ci/build.rb

@@ -42,7 +42,7 @@ module Ci
 
     before_save :update_artifacts_size, if: :artifacts_file_changed?
     before_save :ensure_token
-    before_destroy { project }
+    before_destroy { unscoped_project }
 
     after_create :execute_hooks
     after_save :update_project_statistics, if: :artifacts_size_changed?
@@ -423,16 +423,23 @@ module Ci
     # This method returns old path to artifacts only if it already exists.
     #
     def artifacts_path
+      # We need the project even if it's soft deleted, because whenever
+      # we're really deleting the project, we'll also delete the builds,
+      # and in order to delete the builds, we need to know where to find
+      # the artifacts, which is depending on the data of the project.
+      # We need to retain the project in this case.
+      the_project = project || unscoped_project
+
       old = File.join(created_at.utc.strftime('%Y_%m'),
-                      project.ci_id.to_s,
+                      the_project.ci_id.to_s,
                       id.to_s)
 
       old_store = File.join(ArtifactUploader.artifacts_path, old)
-      return old if project.ci_id && File.directory?(old_store)
+      return old if the_project.ci_id && File.directory?(old_store)
 
       File.join(
         created_at.utc.strftime('%Y_%m'),
-        project.id.to_s,
+        the_project.id.to_s,
         id.to_s
       )
     end
@@ -563,6 +570,10 @@ module Ci
       self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
     end
 
+    def unscoped_project
+      @unscoped_project ||= Project.unscoped.find_by(id: gl_project_id)
+    end
+
     def predefined_variables
       variables = [
         { key: 'CI', value: 'true', public: true },
@@ -601,6 +612,8 @@ module Ci
     end
 
     def update_project_statistics
+      return unless project
+
       ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
     end
   end

changelogs/unreleased/fix-deleting-project-again.yml

+---
+title: Fix deleting projects with pipelines and builds
+merge_request: 8960
+author:

spec/services/projects/destroy_service_spec.rb

@@ -9,12 +9,27 @@ describe Projects::DestroyService, services: true do
 
   shared_examples 'deleting the project' do
     it 'deletes the project' do
-      expect(Project.all).not_to include(project)
+      expect(Project.unscoped.all).not_to include(project)
       expect(Dir.exist?(path)).to be_falsey
       expect(Dir.exist?(remove_path)).to be_falsey
     end
   end
 
+  shared_examples 'deleting the project with pipeline and build' do
+    context 'with pipeline and build' do # which has optimistic locking
+      let!(:pipeline) { create(:ci_pipeline, project: project) }
+      let!(:build) { create(:ci_build, :artifacts, pipeline: pipeline) }
+
+      before do
+        perform_enqueued_jobs do
+          destroy_project(project, user, {})
+        end
+      end
+
+      it_behaves_like 'deleting the project'
+    end
+  end
+
   context 'Sidekiq inline' do
     before do
       # Run sidekiq immediatly to check that renamed repository will be removed
@@ -48,30 +63,24 @@ describe Projects::DestroyService, services: true do
     it { expect(Dir.exist?(remove_path)).to be_truthy }
   end
 
-  context 'async delete of project with private issue visibility' do
-    let!(:async) { true }
+  context 'with async_execute' do
+    let(:async) { true }
 
-    before do
-      project.project_feature.update_attribute("issues_access_level", ProjectFeature::PRIVATE)
-      # Run sidekiq immediately to check that renamed repository will be removed
-      Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
+    context 'async delete of project with private issue visibility' do
+      before do
+        project.project_feature.update_attribute("issues_access_level", ProjectFeature::PRIVATE)
+        # Run sidekiq immediately to check that renamed repository will be removed
+        Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
+      end
+
+      it_behaves_like 'deleting the project'
     end
 
-    it_behaves_like 'deleting the project'
+    it_behaves_like 'deleting the project with pipeline and build'
   end
 
-  context 'delete with pipeline' do # which has optimistic locking
-    let!(:pipeline) { create(:ci_pipeline, project: project) }
-
-    before do
-      expect(project).to receive(:destroy!).and_call_original
-
-      perform_enqueued_jobs do
-        destroy_project(project, user, {})
-      end
-    end
-
-    it_behaves_like 'deleting the project'
+  context 'with execute' do
+    it_behaves_like 'deleting the project with pipeline and build'
   end
 
   context 'container registry' do