Commit 938e90f4 authored by Thong Kuah's avatar Thong Kuah Committed by Stan Hu

Services to uninstall cluster application

+ to monitor progress of uninstallation pod
parent 33a765c1
# frozen_string_literal: true
module Clusters
module Applications
class CheckUninstallProgressService < BaseHelmService
def execute
return unless app.uninstalling?
case installation_phase
when Gitlab::Kubernetes::Pod::SUCCEEDED
on_success
when Gitlab::Kubernetes::Pod::FAILED
on_failed
else
check_timeout
end
rescue Kubeclient::HttpError => e
log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}")
end
private
def on_success
app.make_uninstalled!
ensure
remove_installation_pod
end
def on_failed
app.make_errored!("Operation failed. Check pod logs for #{pod_name} for more details.")
end
def check_timeout
if timeouted?
begin
app.make_errored!("Operation timed out. Check pod logs for #{pod_name} for more details.")
end
else
WaitForUninstallAppWorker.perform_in(WaitForUninstallAppWorker::INTERVAL, app.name, app.id)
end
end
def pod_name
app.uninstall_command.pod_name
end
def timeouted?
Time.now.utc - app.updated_at.to_time.utc > WaitForUninstallAppWorker::TIMEOUT
end
def remove_installation_pod
helm_api.delete_pod!(pod_name)
end
def installation_phase
helm_api.status(pod_name)
end
end
end
end
# frozen_string_literal: true
module Clusters
module Applications
class UninstallService < BaseHelmService
def execute
return unless app.scheduled?
app.make_uninstalling!
uninstall
end
private
def uninstall
helm_api.uninstall(app.uninstall_command)
Clusters::Applications::WaitForUninstallAppWorker.perform_in(
Clusters::Applications::WaitForUninstallAppWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}")
rescue StandardError => e
log_error(e)
app.make_errored!('Failed to uninstall.')
end
end
end
end
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
- gcp_cluster:cluster_wait_for_ingress_ip_address - gcp_cluster:cluster_wait_for_ingress_ip_address
- gcp_cluster:cluster_configure - gcp_cluster:cluster_configure
- gcp_cluster:cluster_project_configure - gcp_cluster:cluster_project_configure
- gcp_cluster:clusters_applications_wait_for_uninstall_app
- github_import_advance_stage - github_import_advance_stage
- github_importer:github_import_import_diff_note - github_importer:github_import_import_diff_note
......
# frozen_string_literal: true
module Clusters
module Applications
class WaitForUninstallAppWorker
include ApplicationWorker
include ClusterQueue
include ClusterApplications
INTERVAL = 10.seconds
TIMEOUT = 20.minutes
def perform(app_name, app_id)
find_application(app_name, app_id) do |app|
Clusters::Applications::CheckUninstallProgressService.new(app).execute
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Clusters::Applications::CheckUninstallProgressService do
RESCHEDULE_PHASES = Gitlab::Kubernetes::Pod::PHASES - [Gitlab::Kubernetes::Pod::SUCCEEDED, Gitlab::Kubernetes::Pod::FAILED].freeze
let(:application) { create(:clusters_applications_helm, :uninstalling) }
let(:service) { described_class.new(application) }
let(:phase) { Gitlab::Kubernetes::Pod::UNKNOWN }
let(:errors) { nil }
let(:worker_class) { Clusters::Applications::WaitForUninstallAppWorker }
shared_examples 'a not yet terminated installation' do |a_phase|
let(:phase) { a_phase }
before do
expect(service).to receive(:installation_phase).once.and_return(phase)
end
context "when phase is #{a_phase}" do
context 'when not timeouted' do
it 'reschedule a new check' do
expect(worker_class).to receive(:perform_in).once
expect(service).not_to receive(:remove_installation_pod)
expect do
service.execute
application.reload
end.not_to change(application, :status)
expect(application.status_reason).to be_nil
end
end
end
end
before do
allow(service).to receive(:installation_errors).and_return(errors)
allow(service).to receive(:remove_installation_pod).and_return(nil)
end
context 'when application is installing' do
RESCHEDULE_PHASES.each { |phase| it_behaves_like 'a not yet terminated installation', phase }
context 'when installation POD succeeded' do
let(:phase) { Gitlab::Kubernetes::Pod::SUCCEEDED }
before do
expect(service).to receive(:installation_phase).once.and_return(phase)
end
it 'removes the installation POD' do
expect(service).to receive(:remove_installation_pod).once
service.execute
end
it 'make the application installed' do
expect(worker_class).not_to receive(:perform_in)
service.execute
expect(application).to be_uninstalled
expect(application.status_reason).to be_nil
end
end
context 'when installation POD failed' do
let(:phase) { Gitlab::Kubernetes::Pod::FAILED }
let(:errors) { 'test installation failed' }
before do
expect(service).to receive(:installation_phase).once.and_return(phase)
end
it 'make the application errored' do
service.execute
expect(application).to be_uninstall_errored
expect(application.status_reason).to eq('Operation failed. Check pod logs for uninstall-helm for more details.')
end
end
context 'when timed out' do
let(:application) { create(:clusters_applications_helm, :timeouted, :uninstalling) }
before do
expect(service).to receive(:installation_phase).once.and_return(phase)
end
it 'make the application errored' do
expect(worker_class).not_to receive(:perform_in)
service.execute
expect(application).to be_uninstall_errored
expect(application.status_reason).to eq('Operation timed out. Check pod logs for uninstall-helm for more details.')
end
end
context 'when installation raises a Kubeclient::HttpError' do
let(:cluster) { create(:cluster, :provided_by_user, :project) }
let(:logger) { service.send(:logger) }
let(:error) { Kubeclient::HttpError.new(401, 'Unauthorized', nil) }
before do
application.update!(cluster: cluster)
expect(service).to receive(:installation_phase).and_raise(error)
end
include_examples 'logs kubernetes errors' do
let(:error_name) { 'Kubeclient::HttpError' }
let(:error_message) { 'Unauthorized' }
let(:error_code) { 401 }
end
it 'shows the response code from the error' do
service.execute
expect(application).to be_uninstall_errored
expect(application.status_reason).to eq('Kubernetes error: 401')
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Clusters::Applications::UninstallService, '#execute' do
let(:application) { create(:clusters_applications_helm, :scheduled) }
let(:service) { described_class.new(application) }
let(:helm_client) { instance_double(Gitlab::Kubernetes::Helm::Api) }
let(:worker_class) { Clusters::Applications::WaitForUninstallAppWorker }
before do
allow(service).to receive(:helm_api).and_return(helm_client)
end
context 'when there are no errors' do
before do
expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand))
allow(worker_class).to receive(:perform_in).and_return(nil)
end
it 'make the application to be uninstalling' do
expect(application.cluster).not_to be_nil
service.execute
expect(application).to be_uninstalling
end
it 'schedule async installation status check' do
expect(worker_class).to receive(:perform_in).once
service.execute
end
end
context 'when k8s cluster communication fails' do
let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
before do
expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand)).and_raise(error)
end
include_examples 'logs kubernetes errors' do
let(:error_name) { 'Kubeclient::HttpError' }
let(:error_message) { 'system failure' }
let(:error_code) { 500 }
end
it 'make the application errored' do
service.execute
expect(application).to be_uninstall_errored
expect(application.status_reason).to match('Kubernetes error: 500')
end
end
context 'a non kubernetes error happens' do
let(:application) { create(:clusters_applications_helm, :scheduled) }
let(:error) { StandardError.new('something bad happened') }
before do
expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand)).and_raise(error)
end
include_examples 'logs kubernetes errors' do
let(:error_name) { 'StandardError' }
let(:error_message) { 'something bad happened' }
let(:error_code) { nil }
end
it 'make the application errored' do
service.execute
expect(application).to be_uninstall_errored
expect(application.status_reason).to eq('Failed to uninstall.')
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Clusters::Applications::WaitForUninstallAppWorker, '#perform' do
let(:app) { create(:clusters_applications_helm) }
let(:app_name) { app.name }
let(:app_id) { app.id }
subject { described_class.new.perform(app_name, app_id) }
context 'app exists' do
let(:service) { instance_double(Clusters::Applications::CheckUninstallProgressService) }
it 'calls the check service' do
expect(Clusters::Applications::CheckUninstallProgressService).to receive(:new).with(app).and_return(service)
expect(service).to receive(:execute).once
subject
end
end
context 'app does not exist' do
let(:app_id) { 0 }
it 'does not call the check service' do
expect(Clusters::Applications::CheckUninstallProgressService).not_to receive(:new)
expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment