Add option to delete cached Kubernetes namespaces

If a user modifies their Kubernetes cluster directly
it can become out of sync with what GitLab has stored,
which can cause deploys to fail.

By deleting these records we allow them to be fetched
from the cluster on the next deploy, which brings
GitLab back in sync with the cluster.

https://gitlab.com/gitlab-org/gitlab/merge_requests/20411

app/controllers/clusters/clusters_controller.rb

@@ -3,14 +3,14 @@
 class Clusters::ClustersController < Clusters::BaseController
   include RoutableActions
 
-  before_action :cluster, only: [:cluster_status, :show, :update, :destroy]
+  before_action :cluster, only: [:cluster_status, :show, :update, :destroy, :clear_cache]
   before_action :generate_gcp_authorize_url, only: [:new]
   before_action :validate_gcp_token, only: [:new]
   before_action :gcp_cluster, only: [:new]
   before_action :user_cluster, only: [:new]
   before_action :authorize_create_cluster!, only: [:new, :authorize_aws_role, :revoke_aws_role, :aws_proxy]
   before_action :authorize_update_cluster!, only: [:update]
-  before_action :authorize_admin_cluster!, only: [:destroy]
+  before_action :authorize_admin_cluster!, only: [:destroy, :clear_cache]
   before_action :update_applications_status, only: [:cluster_status]
   before_action only: [:new, :create_gcp] do
     push_frontend_feature_flag(:create_eks_clusters)
@@ -169,6 +169,12 @@ class Clusters::ClustersController < Clusters::BaseController
     render json: response.body, status: response.status
   end
 
+  def clear_cache
+    cluster.delete_cached_resources!
+
+    redirect_to cluster.show_path, notice: _('Cluster cache cleared.')
+  end
+
   private
 
   def destroy_params

app/models/clusters/cluster.rb

@@ -267,6 +267,10 @@ module Clusters
       end
     end
 
+    def delete_cached_resources!
+      kubernetes_namespaces.delete_all(:delete_all)
+    end
+
     private
 
     def unique_management_project_environment_scope

app/presenters/clusterable_presenter.rb

@@ -65,6 +65,10 @@ class ClusterablePresenter < Gitlab::View::Presenter::Delegated
     raise NotImplementedError
   end
 
+  def clear_cluster_cache_path(cluster)
+    raise NotImplementedError
+  end
+
   def cluster_path(cluster, params = {})
     raise NotImplementedError
   end

app/presenters/group_clusterable_presenter.rb

@@ -19,6 +19,11 @@ class GroupClusterablePresenter < ClusterablePresenter
     update_applications_group_cluster_path(clusterable, cluster, application)
   end
 
+  override :clear_cluster_cache_path
+  def clear_cluster_cache_path(cluster)
+    clear_cache_group_cluster_path(clusterable, cluster)
+  end
+
   override :cluster_path
   def cluster_path(cluster, params = {})
     group_cluster_path(clusterable, cluster, params)

app/presenters/instance_clusterable_presenter.rb

@@ -37,6 +37,11 @@ class InstanceClusterablePresenter < ClusterablePresenter
     update_applications_admin_cluster_path(cluster, application)
   end
 
+  override :clear_cluster_cache_path
+  def clear_cluster_cache_path(cluster)
+    clear_cache_admin_cluster_path(cluster)
+  end
+
   override :cluster_path
   def cluster_path(cluster, params = {})
     admin_cluster_path(cluster, params)

app/presenters/project_clusterable_presenter.rb

@@ -19,6 +19,11 @@ class ProjectClusterablePresenter < ClusterablePresenter
     update_applications_project_cluster_path(clusterable, cluster, application)
   end
 
+  override :clear_cluster_cache_path
+  def clear_cluster_cache_path(cluster)
+    clear_cache_project_cluster_path(clusterable, cluster)
+  end
+
   override :cluster_path
   def cluster_path(cluster, params = {})
     project_cluster_path(clusterable, cluster, params)

app/views/clusters/clusters/_advanced_settings.html.haml

@@ -28,6 +28,14 @@
     .form-group
       = field.submit _('Save changes'), class: 'btn btn-success qa-save-domain'
 
+  - if @cluster.managed?
+    .sub-section.form-group
+      %h4
+        = s_('ClusterIntegration|Clear cluster cache')
+      %p
+        = s_("ClusterIntegration|Clear the local cache of namespace and service accounts. This is necessary if your integration has become out of sync. The cache is repopulated during the next CI job that requires namespace and service accounts.")
+      = link_to(s_('ClusterIntegration|Clear cluster cache'), clusterable.clear_cluster_cache_path(@cluster), method: :delete, class: 'btn btn-primary')
+
   .sub-section.form-group
     %h4.text-danger
       = s_('ClusterIntegration|Remove Kubernetes cluster integration')

changelogs/unreleased/31759-clear-cluster-cache.yml

+---
+title: Add option to delete cached Kubernetes namespaces
+merge_request: 20411
+author:
+type: added

config/routes.rb

@@ -166,6 +166,7 @@ Rails.application.routes.draw do
         end
 
         get :cluster_status, format: :json
+        delete :clear_cache
       end
     end
   end

doc/user/group/clusters/index.md

@@ -75,6 +75,21 @@ NOTE: **Note:**
 If you [install applications](#installing-applications) on your cluster, GitLab will create
 the resources required to run these even if you have chosen to manage your own cluster.
 
+### Clearing the cluster cache
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31759) in GitLab 12.6.
+
+If you choose to allow GitLab to manage your cluster for you, GitLab stores a cached
+version of the namespaces and service accounts it creates for your projects. If you
+modify these resources in your cluster manually, this cache can fall out of sync with
+your cluster, which can cause deployment jobs to fail.
+
+To clear the cache:
+
+1. Navigate to your group’s **Kubernetes** page, and select your cluster.
+1. Expand the **Advanced settings** section.
+1. Click **Clear cluster cache**.
+
 ## Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/24580) in GitLab 11.8.

doc/user/project/clusters/index.md

@@ -132,6 +132,21 @@ NOTE: **Note:**
 If you [install applications](#installing-applications) on your cluster, GitLab will create
 the resources required to run these even if you have chosen to manage your own cluster.
 
+#### Clearing the cluster cache
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31759) in GitLab 12.6.
+
+If you choose to allow GitLab to manage your cluster for you, GitLab stores a cached
+version of the namespaces and service accounts it creates for your projects. If you
+modify these resources in your cluster manually, this cache can fall out of sync with
+your cluster, which can cause deployment jobs to fail.
+
+To clear the cache:
+
+1. Navigate to your project’s **Operations > Kubernetes** page, and select your cluster.
+1. Expand the **Advanced settings** section.
+1. Click **Clear cluster cache**.
+
 ### Base domain
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/24580) in GitLab 11.8.

locale/gitlab.pot

@@ -3469,6 +3469,9 @@ msgstr ""
 msgid "Cluster Health"
 msgstr ""
 
+msgid "Cluster cache cleared."
+msgstr ""
+
 msgid "ClusterIntegration| %{custom_domain_start}More information%{custom_domain_end}."
 msgstr ""
 
@@ -3601,6 +3604,12 @@ msgstr ""
 msgid "ClusterIntegration|Choose which of your environments will use this cluster."
 msgstr ""
 
+msgid "ClusterIntegration|Clear cluster cache"
+msgstr ""
+
+msgid "ClusterIntegration|Clear the local cache of namespace and service accounts. This is necessary if your integration has become out of sync. The cache is repopulated during the next CI job that requires namespace and service accounts."
+msgstr ""
+
 msgid "ClusterIntegration|Cloud Run"
 msgstr ""
 

spec/controllers/admin/clusters_controller_spec.rb

@@ -448,6 +448,33 @@ describe Admin::ClustersController do
     end
   end
 
+  describe 'DELETE clear cluster cache' do
+    let(:cluster) { create(:cluster, :instance) }
+    let!(:kubernetes_namespace) do
+      create(:cluster_kubernetes_namespace,
+        cluster: cluster,
+        project: create(:project)
+      )
+    end
+
+    def go
+      delete :clear_cache, params: { id: cluster }
+    end
+
+    it 'deletes the namespaces associated with the cluster' do
+      expect { go }.to change { Clusters::KubernetesNamespace.count }
+
+      expect(response).to redirect_to(admin_cluster_path(cluster))
+      expect(cluster.kubernetes_namespaces).to be_empty
+    end
+
+    describe 'security' do
+      it { expect { go }.to be_allowed_for(:admin) }
+      it { expect { go }.to be_denied_for(:user) }
+      it { expect { go }.to be_denied_for(:external) }
+    end
+  end
+
   describe 'GET #cluster_status' do
     let(:cluster) { create(:cluster, :providing_by_gcp, :instance) }
 

spec/controllers/groups/clusters_controller_spec.rb

@@ -516,6 +516,42 @@ describe Groups::ClustersController do
     end
   end
 
+  describe 'DELETE clear cluster cache' do
+    let(:cluster) { create(:cluster, :group, groups: [group]) }
+    let!(:kubernetes_namespace) do
+      create(:cluster_kubernetes_namespace,
+        cluster: cluster,
+        project: create(:project)
+      )
+    end
+
+    def go
+      delete :clear_cache,
+        params: {
+          group_id: group,
+          id: cluster
+        }
+    end
+
+    it 'deletes the namespaces associated with the cluster' do
+      expect { go }.to change { Clusters::KubernetesNamespace.count }
+
+      expect(response).to redirect_to(group_cluster_path(group, cluster))
+      expect(cluster.kubernetes_namespaces).to be_empty
+    end
+
+    describe 'security' do
+      it { expect { go }.to be_allowed_for(:admin) }
+      it { expect { go }.to be_allowed_for(:owner).of(group) }
+      it { expect { go }.to be_allowed_for(:maintainer).of(group) }
+      it { expect { go }.to be_denied_for(:developer).of(group) }
+      it { expect { go }.to be_denied_for(:reporter).of(group) }
+      it { expect { go }.to be_denied_for(:guest).of(group) }
+      it { expect { go }.to be_denied_for(:user) }
+      it { expect { go }.to be_denied_for(:external) }
+    end
+  end
+
   describe 'GET cluster_status' do
     let(:cluster) { create(:cluster, :providing_by_gcp, cluster_type: :group_type, groups: [group]) }
 

spec/controllers/projects/clusters_controller_spec.rb

@@ -517,6 +517,38 @@ describe Projects::ClustersController do
     end
   end
 
+  describe 'DELETE clear cluster cache' do
+    let(:cluster) { create(:cluster, :project, projects: [project]) }
+    let!(:kubernetes_namespace) { create(:cluster_kubernetes_namespace, cluster: cluster) }
+
+    def go
+      delete :clear_cache,
+        params: {
+          namespace_id: project.namespace,
+          project_id: project,
+          id: cluster
+        }
+    end
+
+    it 'deletes the namespaces associated with the cluster' do
+      expect { go }.to change { Clusters::KubernetesNamespace.count }
+
+      expect(response).to redirect_to(project_cluster_path(project, cluster))
+      expect(cluster.kubernetes_namespaces).to be_empty
+    end
+
+    describe 'security' do
+      it { expect { go }.to be_allowed_for(:admin) }
+      it { expect { go }.to be_allowed_for(:owner).of(project) }
+      it { expect { go }.to be_allowed_for(:maintainer).of(project) }
+      it { expect { go }.to be_denied_for(:developer).of(project) }
+      it { expect { go }.to be_denied_for(:reporter).of(project) }
+      it { expect { go }.to be_denied_for(:guest).of(project) }
+      it { expect { go }.to be_denied_for(:user) }
+      it { expect { go }.to be_denied_for(:external) }
+    end
+  end
+
   describe 'GET cluster_status' do
     let(:cluster) { create(:cluster, :providing_by_gcp, projects: [project]) }
 

spec/models/clusters/cluster_spec.rb

@@ -960,4 +960,20 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
       end
     end
   end
+
+  describe '#delete_cached_resources!' do
+    let!(:cluster) { create(:cluster, :project) }
+    let!(:staging_namespace) { create(:cluster_kubernetes_namespace, cluster: cluster, namespace: 'staging') }
+    let!(:production_namespace) { create(:cluster_kubernetes_namespace, cluster: cluster, namespace: 'production') }
+
+    subject { cluster.delete_cached_resources! }
+
+    it 'deletes associated namespace records' do
+      expect(cluster.kubernetes_namespaces).to match_array([staging_namespace, production_namespace])
+
+      subject
+
+      expect(cluster.kubernetes_namespaces).to be_empty
+    end
+  end
 end

spec/presenters/group_clusterable_presenter_spec.rb

@@ -83,6 +83,12 @@ describe GroupClusterablePresenter do
     it { is_expected.to eq(update_applications_group_cluster_path(group, cluster, application)) }
   end
 
+  describe '#clear_cluster_cache_path' do
+    subject { presenter.clear_cluster_cache_path(cluster) }
+
+    it { is_expected.to eq(clear_cache_group_cluster_path(group, cluster)) }
+  end
+
   describe '#cluster_path' do
     subject { presenter.cluster_path(cluster) }
 

spec/presenters/instance_clusterable_presenter_spec.rb

@@ -34,4 +34,10 @@ describe InstanceClusterablePresenter do
 
     it { is_expected.to eq(aws_proxy_admin_clusters_path(resource: resource)) }
   end
+
+  describe '#clear_cluster_cache_path' do
+    subject { presenter.clear_cluster_cache_path(cluster) }
+
+    it { is_expected.to eq(clear_cache_admin_cluster_path(cluster)) }
+  end
 end

spec/presenters/project_clusterable_presenter_spec.rb

@@ -83,6 +83,12 @@ describe ProjectClusterablePresenter do
     it { is_expected.to eq(update_applications_project_cluster_path(project, cluster, application)) }
   end
 
+  describe '#clear_cluster_cache_path' do
+    subject { presenter.clear_cluster_cache_path(cluster) }
+
+    it { is_expected.to eq(clear_cache_project_cluster_path(project, cluster)) }
+  end
+
   describe '#cluster_path' do
     subject { presenter.cluster_path(cluster) }