Commit 093e9546 authored by Peter Leitzen's avatar Peter Leitzen Committed by Dan Davison

Extract gcloud and minikube cluster provider

Add cluster provider k3d
parent 94490b63
...@@ -359,6 +359,13 @@ module QA ...@@ -359,6 +359,13 @@ module QA
autoload :KubernetesCluster, 'qa/service/kubernetes_cluster' autoload :KubernetesCluster, 'qa/service/kubernetes_cluster'
autoload :Omnibus, 'qa/service/omnibus' autoload :Omnibus, 'qa/service/omnibus'
autoload :Runner, 'qa/service/runner' autoload :Runner, 'qa/service/runner'
module ClusterProvider
autoload :Base, 'qa/service/cluster_provider/base'
autoload :Gcloud, 'qa/service/cluster_provider/gcloud'
autoload :Minikube, 'qa/service/cluster_provider/minikube'
autoload :K3d, 'qa/service/cluster_provider/k3d'
end
end end
## ##
......
# frozen_string_literal: true
module QA
module Service
module ClusterProvider
class Base
include Service::Shellout
attr_reader :rbac
def initialize(rbac:)
@rbac = rbac
end
def cluster_name
@cluster_name ||= "qa-cluster-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}-#{SecureRandom.hex(4)}"
end
def set_credentials(admin_user)
raise NotImplementedError
end
def validate_dependencies
raise NotImplementedError
end
def setup
raise NotImplementedError
end
def teardown
raise NotImplementedError
end
def filter_credentials(credentials)
credentials
end
end
end
end
end
# frozen_string_literal: true
module QA
module Service
module ClusterProvider
class Gcloud < Base
def validate_dependencies
find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.")
end
def set_credentials(admin_user)
master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`)
shell <<~CMD.tr("\n", ' ')
kubectl config set-credentials #{admin_user}
--username #{master_auth['masterAuth']['username']}
--password #{master_auth['masterAuth']['password']}
CMD
end
def setup
login_if_not_already_logged_in
create_cluster
end
def teardown
delete_cluster
end
private
def login_if_not_already_logged_in
if Runtime::Env.has_gcloud_credentials?
attempt_login_with_env_vars
else
account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"`
if account.empty?
raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
puts "gcloud account found. Using: #{account} for creating K8s cluster."
end
end
end
def attempt_login_with_env_vars
puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
gcloud_account_key = Tempfile.new('gcloud-account-key')
gcloud_account_key.write(Runtime::Env.gcloud_account_key)
gcloud_account_key.close
gcloud_account_email = Runtime::Env.gcloud_account_email
shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}")
ensure
gcloud_account_key && gcloud_account_key.unlink
end
def auth_options
"--enable-legacy-authorization" unless rbac
end
def create_cluster
shell <<~CMD.tr("\n", ' ')
gcloud container clusters
create #{cluster_name}
#{auth_options}
--enable-basic-auth
--region #{Runtime::Env.gcloud_region}
--disk-size 10GB
--num-nodes #{Runtime::Env.gcloud_num_nodes}
&& gcloud container clusters
get-credentials
--region #{Runtime::Env.gcloud_region}
#{cluster_name}
CMD
end
def delete_cluster
shell <<~CMD.tr("\n", ' ')
gcloud container clusters delete
--region #{Runtime::Env.gcloud_region}
#{cluster_name}
--quiet --async
CMD
end
end
end
end
end
# frozen_string_literal: true
module QA
module Service
module ClusterProvider
class K3d < Base
def validate_dependencies
find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.")
end
def set_credentials(admin_user)
end
def setup
shell "k3d create --workers 1 --name #{cluster_name} --wait 0"
@old_kubeconfig = ENV['KUBECONFIG']
ENV['KUBECONFIG'] = fetch_kubeconfig
raise "Could not fetch kubeconfig" unless ENV['KUBECONFIG']
install_local_storage
end
def teardown
ENV['KUBECONFIG'] = @old_kubeconfig
shell "k3d delete --name #{cluster_name}"
end
# Fetch "real" certificate
# See https://github.com/rancher/k3s/issues/27
def filter_credentials(credentials)
kubeconfig = YAML.load_file(ENV['KUBECONFIG'])
ca_certificate = kubeconfig.dig('clusters', 0, 'cluster', 'certificate-authority-data')
credentials.merge('data' => credentials['data'].merge('ca.crt' => ca_certificate))
end
private
def retry_until(max_attempts: 10, wait: 1)
max_attempts.times do
result = yield
return result if result
sleep wait
end
raise "Retried #{max_attempts} times. Aborting"
end
def fetch_kubeconfig
retry_until do
config = `k3d get-kubeconfig --name #{cluster_name}`.chomp
config if config =~ /kubeconfig.yaml/
end
end
def install_local_storage
shell('kubectl apply -f -', stdin_data: local_storage_config)
end
# See https://github.com/rancher/k3d/issues/67
def local_storage_config
<<~YAML
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: storage-provisioner
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: storage-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:persistent-volume-provisioner
subjects:
- kind: ServiceAccount
name: storage-provisioner
namespace: kube-system
---
apiVersion: v1
kind: Pod
metadata:
name: storage-provisioner
namespace: kube-system
spec:
serviceAccountName: storage-provisioner
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
hostNetwork: true
containers:
- name: storage-provisioner
image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1
command: ["/storage-provisioner"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /tmp
name: tmp
volumes:
- name: tmp
hostPath:
path: /tmp
type: Directory
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: standard
namespace: kube-system
annotations:
storageclass.kubernetes.io/is-default-class: "true"
labels:
addonmanager.kubernetes.io/mode: EnsureExists
provisioner: k8s.io/minikube-hostpath
YAML
end
end
end
end
end
# frozen_string_literal: true
module QA
module Service
module ClusterProvider
class Minikube < Base
def validate_dependencies
find_executable('minikube') || raise("You must first install `minikube` executable to run these tests.")
end
def set_credentials(admin_user)
end
def setup
shell 'minikube stop'
shell "minikube profile #{cluster_name}"
shell 'minikube start'
end
def teardown
shell 'minikube delete'
end
end
end
end
end
...@@ -9,90 +9,63 @@ module QA ...@@ -9,90 +9,63 @@ module QA
class KubernetesCluster class KubernetesCluster
include Service::Shellout include Service::Shellout
attr_reader :api_url, :ca_certificate, :token, :rbac attr_reader :api_url, :ca_certificate, :token, :rbac, :provider
def initialize(rbac: true) def initialize(rbac: true, provider_class: QA::Service::ClusterProvider::Gcloud)
@rbac = rbac @rbac = rbac
end @provider = provider_class.new(rbac: rbac)
def cluster_name
@cluster_name ||= "qa-cluster-#{SecureRandom.hex(4)}-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}"
end end
def create! def create!
validate_dependencies validate_dependencies
login_if_not_already_logged_in
@provider.validate_dependencies
shell <<~CMD.tr("\n", ' ') @provider.setup
gcloud container clusters
create #{cluster_name} @api_url = fetch_api_url
#{auth_options}
--enable-basic-auth credentials = @provider.filter_credentials(fetch_credentials)
--region #{Runtime::Env.gcloud_region} @ca_certificate = Base64.decode64(credentials.dig('data', 'ca.crt'))
--disk-size 10GB @token = Base64.decode64(credentials.dig('data', 'token'))
--num-nodes #{Runtime::Env.gcloud_num_nodes}
&& gcloud container clusters
get-credentials
--region #{Runtime::Env.gcloud_region}
#{cluster_name}
CMD
@api_url = `kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}'`
@admin_user = "#{cluster_name}-admin"
master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`)
shell <<~CMD.tr("\n", ' ')
kubectl config set-credentials #{@admin_user}
--username #{master_auth['masterAuth']['username']}
--password #{master_auth['masterAuth']['password']}
CMD
if rbac
create_service_account
secrets = JSON.parse(`kubectl get secrets -o json`)
gitlab_account = secrets['items'].find do |item|
item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account'
end
@ca_certificate = Base64.decode64(gitlab_account['data']['ca.crt'])
@token = Base64.decode64(gitlab_account['data']['token'])
else
@ca_certificate = Base64.decode64(`kubectl get secrets -o jsonpath="{.items[0].data['ca\\.crt']}"`)
@token = Base64.decode64(`kubectl get secrets -o jsonpath='{.items[0].data.token}'`)
end
self self
end end
def remove! def remove!
shell <<~CMD.tr("\n", ' ') @provider.teardown
gcloud container clusters delete end
--region #{Runtime::Env.gcloud_region}
#{cluster_name} def cluster_name
--quiet --async @provider.cluster_name
CMD
end end
private private
def create_service_account def fetch_api_url
shell('kubectl create -f -', stdin_data: service_account) `kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'`
shell("kubectl --user #{@admin_user} create -f -", stdin_data: service_account_role_binding) end
def fetch_credentials
return global_credentials unless rbac
@provider.set_credentials(admin_user)
create_service_account(admin_user)
account_credentials
end end
def service_account def admin_user
<<~YAML @admin_user ||= "#{@provider.cluster_name}-admin"
end
def create_service_account(user)
service_account = <<~YAML
---
apiVersion: v1 apiVersion: v1
kind: ServiceAccount kind: ServiceAccount
metadata: metadata:
name: gitlab-account name: gitlab-account
namespace: default namespace: default
YAML ---
end
def service_account_role_binding
<<~YAML
kind: ClusterRoleBinding kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
metadata: metadata:
...@@ -106,39 +79,24 @@ module QA ...@@ -106,39 +79,24 @@ module QA
name: cluster-admin name: cluster-admin
apiGroup: rbac.authorization.k8s.io apiGroup: rbac.authorization.k8s.io
YAML YAML
end
def auth_options shell('kubectl apply -f -', stdin_data: service_account)
"--enable-legacy-authorization" unless rbac
end end
def validate_dependencies def account_credentials
find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.") secrets = JSON.parse(`kubectl get secrets -o json`)
find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.")
end
def login_if_not_already_logged_in secrets['items'].find do |item|
if Runtime::Env.has_gcloud_credentials? item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account'
attempt_login_with_env_vars
else
account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"`
if account.empty?
raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
puts "gcloud account found. Using: #{account} for creating K8s cluster."
end
end end
end end
def attempt_login_with_env_vars def global_credentials
puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." JSON.parse(`kubectl get secrets -o jsonpath='{.items[0]}'`)
gcloud_account_key = Tempfile.new('gcloud-account-key') end
gcloud_account_key.write(Runtime::Env.gcloud_account_key)
gcloud_account_key.close def validate_dependencies
gcloud_account_email = Runtime::Env.gcloud_account_email find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.")
shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}")
ensure
gcloud_account_key && gcloud_account_key.unlink
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment