Add terminals to the Kubernetes deployment service

app/controllers/projects/environments_controller.rb

@@ -56,6 +56,29 @@ class Projects::EnvironmentsController < Projects::ApplicationController
     redirect_to polymorphic_path([project.namespace.becomes(Namespace), project, new_action])
   end
 
+  def terminal
+    # Currently, this acts as a hint to load the terminal details into the cache
+    # if they aren't there already. In the future, users will need these details
+    # to choose between terminals to connect to.
+    @terminals = environment.terminals
+  end
+
+  # GET .../terminal.ws : implemented in gitlab-workhorse
+  def terminal_websocket_authorize
+    Gitlab::Workhorse.verify_api_request!(request.headers)
+
+    # Just return the first terminal for now. If the list is in the process of
+    # being looked up, this may result in a 404 response, so the frontend
+    # should retry
+    terminal = environment.terminals.try(:first)
+    if terminal
+      set_workhorse_internal_api_content_type
+      render json: Gitlab::Workhorse.terminal_websocket(terminal)
+    else
+      render text: 'Not found', status: 404
+    end
+  end
+
   private
 
   def environment_params

app/models/concerns/reactive_caching.rb

@@ -55,6 +55,10 @@ module ReactiveCaching
     self.reactive_cache_refresh_interval = 1.minute
     self.reactive_cache_lifetime = 10.minutes
 
+    def calculate_reactive_cache
+      raise NotImplementedError
+    end
+
     def with_reactive_cache(&blk)
       within_reactive_cache_lifetime do
         data = Rails.cache.read(full_reactive_cache_key)

app/models/environment.rb

@@ -128,6 +128,14 @@ class Environment < ActiveRecord::Base
     end
   end
 
+  def has_terminals?
+    project.deployment_service.present? && available? && last_deployment.present?
+  end
+
+  def terminals
+    project.deployment_service.terminals(self) if has_terminals?
+  end
+
   # An environment name is not necessarily suitable for use in URLs, DNS
   # or other third-party contexts, so provide a slugified version. A slug has
   # the following properties:

app/models/project_services/deployment_service.rb

@@ -12,4 +12,22 @@ class DeploymentService < Service
   def predefined_variables
     []
   end
+
+  # Environments may have a number of terminals. Should return an array of
+  # hashes describing them, e.g.:
+  #
+  #     [{
+  #       :selectors    => {"a" => "b", "foo" => "bar"},
+  #       :url          => "wss://external.example.com/exec",
+  #       :headers      => {"Authorization" => "Token xxx"},
+  #       :subprotocols => ["foo"],
+  #       :ca_pem       => "----BEGIN CERTIFICATE...", # optional
+  #       :created_at   => Time.now.utc
+  #     }]
+  #
+  # Selectors should be a set of values that uniquely identify a particular
+  # terminal
+  def terminals(environment)
+    raise NotImplementedError
+  end
 end

app/models/project_services/kubernetes_service.rb

 class KubernetesService < DeploymentService
+  include Gitlab::Kubernetes
+  include ReactiveCaching
+
+  self.reactive_cache_key = ->(service) { [ service.class.model_name.singular, service.project_id ] }
+
   # Namespace defaults to the project path, but can be overridden in case that
   # is an invalid or inappropriate name
   prop_accessor :namespace
@@ -25,6 +30,8 @@ class KubernetesService < DeploymentService
       length: 1..63
   end
 
+  after_save :clear_reactive_cache!
+
   def initialize_properties
     if properties.nil?
       self.properties = {}
@@ -41,7 +48,8 @@ class KubernetesService < DeploymentService
   end
 
   def help
-    ''
+    'To enable terminal access to Kubernetes environments, label your ' \
+    'deployments with `app=$CI_ENVIRONMENT_SLUG`'
   end
 
   def to_param
@@ -75,9 +83,9 @@ class KubernetesService < DeploymentService
 
   # Check we can connect to the Kubernetes API
   def test(*args)
-    kubeclient = build_kubeclient
-    kubeclient.discover
+    kubeclient = build_kubeclient!
 
+    kubeclient.discover
     { success: kubeclient.discovered, result: "Checked API discovery endpoint" }
   rescue => err
     { success: false, result: err }
@@ -93,20 +101,48 @@ class KubernetesService < DeploymentService
     variables
   end
 
-  private
+  # Constructs a list of terminals from the reactive cache
+  #
+  # Returns nil if the cache is empty, in which case you should try again a
+  # short time later
+  def terminals(environment)
+    with_reactive_cache do |data|
+      pods = data.fetch(:pods, nil)
+      filter_pods(pods, app: environment.slug).
+        flat_map { |pod| terminals_for_pod(api_url, namespace, pod) }.
+        map { |terminal| add_terminal_auth(terminal, token, ca_pem) }
+    end
+  end
 
-  def build_kubeclient(api_path = '/api', api_version = 'v1')
-    return nil unless api_url && namespace && token
+  # Caches all pods in the namespace so other calls don't need to block on
+  # network access.
+  def calculate_reactive_cache
+    return unless active? && project && !project.pending_delete?
 
-    url = URI.parse(api_url)
-    url.path = url.path[0..-2] if url.path[-1] == "/"
-    url.path += api_path
+    kubeclient = build_kubeclient!
+
+    # Store as hashes, rather than as third-party types
+    pods = begin
+      kubeclient.get_pods(namespace: namespace).as_json
+    rescue KubeException => err
+      raise err unless err.error_code == 404
+      []
+    end
+
+    # We may want to cache extra things in the future
+    { pods: pods }
+  end
+
+  private
+
+  def build_kubeclient!(api_path: 'api', api_version: 'v1')
+    raise "Incomplete settings" unless api_url && namespace && token
 
     ::Kubeclient::Client.new(
-      url,
+      join_api_url(api_path),
       api_version,
-      ssl_options: kubeclient_ssl_options,
       auth_options: kubeclient_auth_options,
+      ssl_options: kubeclient_ssl_options,
       http_proxy_uri: ENV['http_proxy']
     )
   end
@@ -125,4 +161,13 @@ class KubernetesService < DeploymentService
   def kubeclient_auth_options
     { bearer_token: token }
   end
+
+  def join_api_url(*parts)
+    url = URI.parse(api_url)
+    prefix = url.path.sub(%r{/+\z}, '')
+
+    url.path = [ prefix, *parts ].join("/")
+
+    url.to_s
+  end
 end

app/serializers/environment_entity.rb

@@ -8,6 +8,7 @@ class EnvironmentEntity < Grape::Entity
   expose :environment_type
   expose :last_deployment, using: DeploymentEntity
   expose :stoppable?
+  expose :has_terminals?, as: :has_terminals
 
   expose :environment_path do |environment|
     namespace_project_environment_path(
@@ -23,5 +24,12 @@ class EnvironmentEntity < Grape::Entity
       environment)
   end
 
+  expose :terminal_path, if: ->(environment, _) { environment.has_terminals? } do |environment|
+    terminal_namespace_project_environment_path(
+      environment.project.namespace,
+      environment.project,
+      environment)
+  end
+
   expose :created_at, :updated_at
 end

lib/gitlab/kubernetes.rb

+module Gitlab
+  # Helper methods to do with Kubernetes network services & resources
+  module Kubernetes
+    # This is the comand that is run to start a terminal session. Kubernetes
+    # expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
+    EXEC_COMMAND = URI.encode_www_form(
+      ['sh', '-c', 'bash || sh'].map { |value| ['command', value] }
+    )
+
+    # Filters an array of pods (as returned by the kubernetes API) by their labels
+    def filter_pods(pods, labels = {})
+      pods.select do |pod|
+        metadata = pod.fetch("metadata", {})
+        pod_labels = metadata.fetch("labels", nil)
+        next unless pod_labels
+
+        labels.all? { |k, v| pod_labels[k.to_s] == v }
+      end
+    end
+
+    # Converts a pod (as returned by the kubernetes API) into a terminal
+    def terminals_for_pod(api_url, namespace, pod)
+      metadata = pod.fetch("metadata", {})
+      status   = pod.fetch("status", {})
+      spec     = pod.fetch("spec", {})
+
+      containers = spec["containers"]
+      pod_name   = metadata["name"]
+      phase      = status["phase"]
+
+      return unless containers.present? && pod_name.present? && phase == "Running"
+
+      created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil
+
+      containers.map do |container|
+        {
+          selectors:    { pod: pod_name, container: container["name"] },
+          url:          container_exec_url(api_url, namespace, pod_name, container["name"]),
+          subprotocols: ['channel.k8s.io'],
+          headers:      Hash.new { |h, k| h[k] = [] },
+          created_at:   created_at,
+        }
+      end
+    end
+
+    def add_terminal_auth(terminal, token, ca_pem = nil)
+      terminal[:headers]['Authorization'] << "Bearer #{token}"
+      terminal[:ca_pem] = ca_pem if ca_pem.present?
+      terminal
+    end
+
+    def container_exec_url(api_url, namespace, pod_name, container_name)
+      url = URI.parse(api_url)
+      url.path = [
+        url.path.sub(%r{/+\z}, ''),
+        'api', 'v1',
+        'namespaces', ERB::Util.url_encode(namespace),
+        'pods', ERB::Util.url_encode(pod_name),
+        'exec'
+      ].join('/')
+
+      url.query = {
+        container: container_name,
+        tty: true,
+        stdin: true,
+        stdout: true,
+        stderr: true,
+      }.to_query + '&' + EXEC_COMMAND
+
+      case url.scheme
+      when 'http'
+        url.scheme = 'ws'
+      when 'https'
+        url.scheme = 'wss'
+      end
+
+      url.to_s
+    end
+  end
+end

lib/gitlab/workhorse.rb

@@ -95,6 +95,19 @@ module Gitlab
         ]
       end
 
+      def terminal_websocket(terminal)
+        details = {
+          'Terminal' => {
+            'Subprotocols' => terminal[:subprotocols],
+            'Url' => terminal[:url],
+            'Header' => terminal[:headers]
+          }
+        }
+        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem)
+
+        details
+      end
+
       def version
         path = Rails.root.join(VERSION_FILE)
         path.readable? ? path.read.chomp : 'unknown'

spec/controllers/projects/environments_controller_spec.rb

@@ -71,6 +71,74 @@ describe Projects::EnvironmentsController do
     end
   end
 
+  describe 'GET #terminal' do
+    context 'with valid id' do
+      it 'responds with a status code 200' do
+        get :terminal, environment_params
+
+        expect(response).to have_http_status(200)
+      end
+
+      it 'loads the terminals for the enviroment' do
+        expect_any_instance_of(Environment).to receive(:terminals)
+
+        get :terminal, environment_params
+      end
+    end
+
+    context 'with invalid id' do
+      it 'responds with a status code 404' do
+        get :terminal, environment_params(id: 666)
+
+        expect(response).to have_http_status(404)
+      end
+    end
+  end
+
+  describe 'GET #terminal_websocket_authorize' do
+    context 'with valid workhorse signature' do
+      before do
+        allow(Gitlab::Workhorse).to receive(:verify_api_request!).and_return(nil)
+      end
+
+      context 'and valid id' do
+        it 'returns the first terminal for the environment' do
+          expect_any_instance_of(Environment).
+            to receive(:terminals).
+            and_return([:fake_terminal])
+
+          expect(Gitlab::Workhorse).
+            to receive(:terminal_websocket).
+            with(:fake_terminal).
+            and_return(workhorse: :response)
+
+          get :terminal_websocket_authorize, environment_params
+
+          expect(response).to have_http_status(200)
+          expect(response.headers["Content-Type"]).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+          expect(response.body).to eq('{"workhorse":"response"}')
+        end
+      end
+
+      context 'and invalid id' do
+        it 'returns 404' do
+          get :terminal_websocket_authorize, environment_params(id: 666)
+          expect(response).to have_http_status(404)
+        end
+      end
+    end
+
+    context 'with invalid workhorse signature' do
+      it 'aborts with an exception' do
+        allow(Gitlab::Workhorse).to receive(:verify_api_request!).and_raise(JWT::DecodeError)
+
+        expect { get :terminal_websocket_authorize, environment_params }.to raise_error(JWT::DecodeError)
+        # controller tests don't set the response status correctly. It's enough
+        # to check that the action raised an exception
+      end
+    end
+  end
+
   def environment_params(opts = {})
     opts.reverse_merge(namespace_id: project.namespace,
                        project_id: project,

spec/factories/projects.rb

@@ -140,7 +140,7 @@ FactoryGirl.define do
         active: true,
         properties: {
           namespace: project.path,
-          api_url: 'https://kubernetes.example.com/api',
+          api_url: 'https://kubernetes.example.com',
           token: 'a' * 40,
         }
       )

spec/lib/gitlab/kubernetes_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Kubernetes do
+  include described_class
+
+  describe '#container_exec_url' do
+    let(:api_url) { 'https://example.com' }
+    let(:namespace) { 'default' }
+    let(:pod_name) { 'pod1' }
+    let(:container_name) { 'container1' }
+
+    subject(:result) { URI::parse(container_exec_url(api_url, namespace, pod_name, container_name)) }
+
+    it { expect(result.scheme).to eq('wss') }
+    it { expect(result.host).to eq('example.com') }
+    it { expect(result.path).to eq('/api/v1/namespaces/default/pods/pod1/exec') }
+    it { expect(result.query).to eq('container=container1&stderr=true&stdin=true&stdout=true&tty=true&command=sh&command=-c&command=bash+%7C%7C+sh') }
+
+    context 'with a HTTP API URL' do
+      let(:api_url) { 'http://example.com' }
+
+      it { expect(result.scheme).to eq('ws') }
+    end
+
+    context 'with a path prefix in the API URL' do
+      let(:api_url) { 'https://example.com/prefix/' }
+      it { expect(result.path).to eq('/prefix/api/v1/namespaces/default/pods/pod1/exec') }
+    end
+
+    context 'with arguments that need urlencoding' do
+      let(:namespace) { 'default namespace' }
+      let(:pod_name) { 'pod 1' }
+      let(:container_name) { 'container 1' }
+
+      it { expect(result.path).to eq('/api/v1/namespaces/default%20namespace/pods/pod%201/exec') }
+      it { expect(result.query).to match(/\Acontainer=container\+1&/) }
+    end
+  end
+end

spec/lib/gitlab/workhorse_spec.rb

@@ -37,6 +37,42 @@ describe Gitlab::Workhorse, lib: true do
     end
   end
 
+  describe '.terminal_websocket' do
+    def terminal(ca_pem: nil)
+      out = {
+        subprotocols: ['foo'],
+        url: 'wss://example.com/terminal.ws',
+        headers: { 'Authorization' => ['Token x'] }
+      }
+      out[:ca_pem] = ca_pem if ca_pem
+      out
+    end
+
+    def workhorse(ca_pem: nil)
+      out = {
+        'Terminal' => {
+          'Subprotocols' => ['foo'],
+          'Url' => 'wss://example.com/terminal.ws',
+          'Header' => { 'Authorization' => ['Token x'] }
+        }
+      }
+      out['Terminal']['CAPem'] = ca_pem if ca_pem
+      out
+    end
+
+    context 'without ca_pem' do
+      subject { Gitlab::Workhorse.terminal_websocket(terminal) }
+
+      it { is_expected.to eq(workhorse) }
+    end
+
+    context 'with ca_pem' do
+      subject { Gitlab::Workhorse.terminal_websocket(terminal(ca_pem: "foo")) }
+
+      it { is_expected.to eq(workhorse(ca_pem: "foo")) }
+    end
+  end
+
   describe '.send_git_diff' do
     let(:diff_refs) { double(base_sha: "base", head_sha: "head") }
     subject { described_class.send_git_patch(repository, diff_refs) }

spec/models/environment_spec.rb

 require 'spec_helper'
 
 describe Environment, models: true do
-  subject(:environment) { create(:environment) }
+  let(:project) { create(:empty_project) }
+  subject(:environment) { create(:environment, project: project) }
 
   it { is_expected.to belong_to(:project) }
   it { is_expected.to have_many(:deployments) }
@@ -31,6 +32,8 @@ describe Environment, models: true do
   end
 
   describe '#includes_commit?' do
+    let(:project) { create(:project) }
+
     context 'without a last deployment' do
       it "returns false" do
         expect(environment.includes_commit?('HEAD')).to be false
@@ -38,9 +41,6 @@ describe Environment, models: true do
     end
 
     context 'with a last deployment' do
-      let(:project)     { create(:project) }
-      let(:environment) { create(:environment, project: project) }
-
       let!(:deployment) do
         create(:deployment, environment: environment, sha: project.commit('master').id)
       end
@@ -65,7 +65,6 @@ describe Environment, models: true do
 
   describe '#first_deployment_for' do
     let(:project)       { create(:project) }
-    let!(:environment)  { create(:environment, project: project) }
     let!(:deployment)   { create(:deployment, environment: environment, ref: commit.parent.id) }
     let!(:deployment1)  { create(:deployment, environment: environment, ref: commit.id) }
     let(:head_commit)   { project.commit }
@@ -196,6 +195,57 @@ describe Environment, models: true do
     end
   end
 
+  describe '#has_terminals?' do
+    subject { environment.has_terminals? }
+
+    context 'when the enviroment is available' do
+      context 'with a deployment service' do
+        let(:project) { create(:kubernetes_project) }
+
+        context 'and a deployment' do
+          let!(:deployment) { create(:deployment, environment: environment) }
+          it { is_expected.to be_truthy }
+        end
+
+        context 'but no deployments' do
+          it { is_expected.to be_falsy }
+        end
+      end
+
+      context 'without a deployment service' do
+        it { is_expected.to be_falsy }
+      end
+    end
+
+    context 'when the environment is unavailable' do
+      let(:project) { create(:kubernetes_project) }
+      before { environment.stop }
+      it { is_expected.to be_falsy }
+    end
+  end
+
+  describe '#terminals' do
+    let(:project) { create(:kubernetes_project) }
+    subject { environment.terminals }
+
+    context 'when the environment has terminals' do
+      before { allow(environment).to receive(:has_terminals?).and_return(true) }
+
+      it 'returns the terminals from the deployment service' do
+        expect(project.deployment_service).
+          to receive(:terminals).with(environment).
+          and_return(:fake_terminals)
+
+        is_expected.to eq(:fake_terminals)
+      end
+    end
+
+    context 'when the environment does not have terminals' do
+      before { allow(environment).to receive(:has_terminals?).and_return(false) }
+      it { is_expected.to eq(nil) }
+    end
+  end
+
   describe '#slug' do
     it "is automatically generated" do
       expect(environment.slug).not_to be_nil

spec/models/project_services/kubernetes_service_spec.rb

 require 'spec_helper'
 
-describe KubernetesService, models: true do
-  let(:project) { create(:empty_project) }
+describe KubernetesService, models: true, caching: true do
+  include KubernetesHelpers
+  include ReactiveCachingHelpers
+
+  let(:project) { create(:kubernetes_project) }
+  let(:service) { project.kubernetes_service }
+
+  # We use Kubeclient to interactive with the Kubernetes API. It will
+  # GET /api/v1 for a list of resources the API supports. This must be stubbed
+  # in addition to any other HTTP requests we expect it to perform.
+  let(:discovery_url) { service.api_url + '/api/v1' }
+  let(:discovery_response) { { body: kube_discovery_body.to_json } }
+
+  let(:pods_url) { service.api_url + "/api/v1/namespaces/#{service.namespace}/pods" }
+  let(:pods_response) { { body: kube_pods_body(kube_pod).to_json } }
+
+  def stub_kubeclient_discover
+    WebMock.stub_request(:get, discovery_url).to_return(discovery_response)
+  end
+
+  def stub_kubeclient_pods
+    stub_kubeclient_discover
+    WebMock.stub_request(:get, pods_url).to_return(pods_response)
+  end
 
   describe "Associations" do
     it { is_expected.to belong_to :project }
@@ -65,22 +87,15 @@ describe KubernetesService, models: true do
   end
 
   describe '#test' do
-    let(:project) { create(:kubernetes_project) }
-    let(:service) { project.kubernetes_service }
-    let(:discovery_url) { service.api_url + '/api/v1' }
-
-    # JSON response body from Kubernetes GET /api/v1 request
-    let(:discovery_response) { { "kind" => "APIResourceList", "groupVersion" => "v1", "resources" => [] }.to_json }
+    before do
+      stub_kubeclient_discover
+    end
 
     context 'with path prefix in api_url' do
       let(:discovery_url) { 'https://kubernetes.example.com/prefix/api/v1' }
 
-      before do
-        service.api_url = 'https://kubernetes.example.com/prefix/'
-      end
-
       it 'tests with the prefix' do
-        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
+        service.api_url = 'https://kubernetes.example.com/prefix/'
 
         expect(service.test[:success]).to be_truthy
         expect(WebMock).to have_requested(:get, discovery_url).once
@@ -88,17 +103,12 @@ describe KubernetesService, models: true do
     end
 
     context 'with custom CA certificate' do
-      let(:certificate) { "CA PEM DATA" }
-      before do
-        service.update_attributes!(ca_pem: certificate)
-      end
-
       it 'is added to the certificate store' do
-        cert = double("certificate")
+        service.ca_pem = "CA PEM DATA"
 
-        expect(OpenSSL::X509::Certificate).to receive(:new).with(certificate).and_return(cert)
+        cert = double("certificate")
+        expect(OpenSSL::X509::Certificate).to receive(:new).with(service.ca_pem).and_return(cert)
         expect_any_instance_of(OpenSSL::X509::Store).to receive(:add_cert).with(cert)
-        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
 
         expect(service.test[:success]).to be_truthy
         expect(WebMock).to have_requested(:get, discovery_url).once
@@ -107,17 +117,15 @@ describe KubernetesService, models: true do
 
     context 'success' do
       it 'reads the discovery endpoint' do
-        WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
-
         expect(service.test[:success]).to be_truthy
         expect(WebMock).to have_requested(:get, discovery_url).once
       end
     end
 
     context 'failure' do
-      it 'fails to read the discovery endpoint' do
-        WebMock.stub_request(:get, discovery_url).to_return(status: 404)
+      let(:discovery_response) { { status: 404 } }
 
+      it 'fails to read the discovery endpoint' do
         expect(service.test[:success]).to be_falsy
         expect(WebMock).to have_requested(:get, discovery_url).once
       end
@@ -156,4 +164,55 @@ describe KubernetesService, models: true do
       )
     end
   end
+
+  describe '#terminals' do
+    let(:environment) { build(:environment, project: project, name: "env", slug: "env-000000") }
+    subject { service.terminals(environment) }
+
+    context 'with invalid pods' do
+      it 'returns no terminals' do
+        stub_reactive_cache(service, pods: [ { "bad" => "pod" } ])
+
+        is_expected.to be_empty
+      end
+    end
+
+    context 'with valid pods' do
+      let(:pod) { kube_pod(app: environment.slug) }
+      let(:terminals) { kube_terminals(service, pod) }
+
+      it 'returns terminals' do
+        stub_reactive_cache(service, pods: [ pod, pod, kube_pod(app: "should-be-filtered-out") ])
+
+        is_expected.to eq(terminals + terminals)
+      end
+    end
+  end
+
+  describe '#calculate_reactive_cache' do
+    before { stub_kubeclient_pods }
+    subject { service.calculate_reactive_cache }
+
+    context 'when service is inactive' do
+      before { service.active = false }
+
+      it { is_expected.to be_nil }
+    end
+
+    context 'when kubernetes responds with valid pods' do
+      it { is_expected.to eq(pods: [kube_pod]) }
+    end
+
+    context 'when kubernetes responds with 500' do
+      let(:pods_response) { { status: 500 } }
+
+      it { expect { subject }.to raise_error(KubeException) }
+    end
+
+    context 'when kubernetes responds with 404' do
+      let(:pods_response) { { status: 404 } }
+
+      it { is_expected.to eq(pods: []) }
+    end
+  end
 end

spec/support/kubernetes_helpers.rb

+module KubernetesHelpers
+  include Gitlab::Kubernetes
+
+  def kube_discovery_body
+    { "kind" => "APIResourceList",
+      "resources" => [
+        { "name" => "pods", "namespaced" => true, "kind" => "Pod" },
+      ],
+    }
+  end
+
+  def kube_pods_body(*pods)
+    { "kind" => "PodList",
+      "items" => [ kube_pod ],
+    }
+  end
+
+  # This is a partial response, it will have many more elements in reality but
+  # these are the ones we care about at the moment
+  def kube_pod(app: "valid-pod-label")
+    { "metadata" => {
+        "name" => "kube-pod",
+        "creationTimestamp" => "2016-11-25T19:55:19Z",
+        "labels" => { "app" => app },
+      },
+      "spec" => {
+        "containers" => [
+          { "name" => "container-0" },
+          { "name" => "container-1" },
+        ],
+      },
+      "status" => { "phase" => "Running" },
+    }
+  end
+
+  def kube_terminals(service, pod)
+    pod_name = pod['metadata']['name']
+    containers = pod['spec']['containers']
+
+    containers.map do |container|
+      terminal = {
+        selectors: { pod: pod_name, container: container['name'] },
+        url:  container_exec_url(service.api_url, service.namespace, pod_name, container['name']),
+        subprotocols: ['channel.k8s.io'],
+        headers: { 'Authorization' => ["Bearer #{service.token}"] },
+        created_at: DateTime.parse(pod['metadata']['creationTimestamp'])
+      }
+      terminal[:ca_pem] = service.ca_pem if service.ca_pem.present?
+      terminal
+    end
+  end
+end