Merge branch 'mattkasa/gitlab-33559-serverless-domains-mtls' into 'master'

mTLS for Serverless Domains

See merge request gitlab-org/gitlab!22408

app/models/clusters/applications/knative.rb

@@ -74,7 +74,7 @@ module Clusters
       end
 
       def ingress_service
-        cluster.kubeclient.get_service('istio-ingressgateway', 'istio-system')
+        cluster.kubeclient.get_service('istio-ingressgateway', Clusters::Kubernetes::ISTIO_SYSTEM_NAMESPACE)
       end
 
       def uninstall_command

app/models/serverless/domain_cluster.rb

@@ -10,6 +10,11 @@ module Serverless
     belongs_to :knative, class_name: 'Clusters::Applications::Knative', foreign_key: 'clusters_applications_knative_id'
     belongs_to :creator, class_name: 'User', optional: true
 
+    attr_encrypted :key,
+      mode: :per_attribute_iv,
+      key: Settings.attr_encrypted_db_key_base_truncated,
+      algorithm: 'aes-256-gcm'
+
     validates :pages_domain, :knative, presence: true
     validates :uuid, presence: true, uniqueness: true, length: { is: Gitlab::Serverless::Domain::UUID_LENGTH },
               format: { with: HEX_REGEXP, message: 'only allows hex characters' }

app/services/clusters/kubernetes.rb

@@ -12,5 +12,7 @@ module Clusters
     GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME = 'gitlab-knative-serving-rolebinding'
     GITLAB_CROSSPLANE_DATABASE_ROLE_NAME = 'gitlab-crossplane-database-role'
     GITLAB_CROSSPLANE_DATABASE_ROLE_BINDING_NAME = 'gitlab-crossplane-database-rolebinding'
+    KNATIVE_SERVING_NAMESPACE = 'knative-serving'
+    ISTIO_SYSTEM_NAMESPACE = 'istio-system'
   end
 end

app/services/clusters/kubernetes/configure_istio_ingress_service.rb

+# frozen_string_literal: true
+
+require 'openssl'
+
+module Clusters
+  module Kubernetes
+    class ConfigureIstioIngressService
+      PASSTHROUGH_RESOURCE = Kubeclient::Resource.new(
+        mode: 'PASSTHROUGH'
+      ).freeze
+
+      MTLS_RESOURCE = Kubeclient::Resource.new(
+        mode: 'MUTUAL',
+        privateKey: '/etc/istio/ingressgateway-certs/tls.key',
+        serverCertificate: '/etc/istio/ingressgateway-certs/tls.crt',
+        caCertificates: '/etc/istio/ingressgateway-ca-certs/cert.pem'
+      ).freeze
+
+      def initialize(cluster:)
+        @cluster = cluster
+        @platform = cluster.platform
+        @kubeclient = platform.kubeclient
+        @knative = cluster.application_knative
+      end
+
+      def execute
+        return configure_certificates if serverless_domain_cluster
+
+        configure_passthrough
+      end
+
+      private
+
+      attr_reader :cluster, :platform, :kubeclient, :knative
+
+      def serverless_domain_cluster
+        knative&.serverless_domain_cluster
+      end
+
+      def configure_certificates
+        create_or_update_istio_cert_and_key
+        set_gateway_wildcard_https(MTLS_RESOURCE)
+      end
+
+      def create_or_update_istio_cert_and_key
+        name = OpenSSL::X509::Name.parse("CN=#{knative.hostname}")
+
+        key = OpenSSL::PKey::RSA.new(2048)
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.version = 2
+        cert.serial = 0
+        cert.not_before = Time.now
+        cert.not_after = Time.now + 1000.years
+
+        cert.public_key = key.public_key
+        cert.subject = name
+        cert.issuer = name
+        cert.sign(key, OpenSSL::Digest::SHA256.new)
+
+        serverless_domain_cluster.update!(
+          key: key.to_pem,
+          certificate: cert.to_pem
+        )
+
+        kubeclient.create_or_update_secret(istio_ca_certs_resource)
+        kubeclient.create_or_update_secret(istio_certs_resource)
+      end
+
+      def istio_ca_certs_resource
+        Gitlab::Kubernetes::GenericSecret.new(
+          'istio-ingressgateway-ca-certs',
+          {
+            'cert.pem': Base64.strict_encode64(serverless_domain_cluster.certificate)
+          },
+          Clusters::Kubernetes::ISTIO_SYSTEM_NAMESPACE
+        ).generate
+      end
+
+      def istio_certs_resource
+        Gitlab::Kubernetes::TlsSecret.new(
+          'istio-ingressgateway-certs',
+          serverless_domain_cluster.certificate,
+          serverless_domain_cluster.key,
+          Clusters::Kubernetes::ISTIO_SYSTEM_NAMESPACE
+        ).generate
+      end
+
+      def set_gateway_wildcard_https(tls_resource)
+        gateway_resource = gateway
+        gateway_resource.spec.servers.each do |server|
+          next unless server.hosts == ['*'] && server.port.name == 'https'
+
+          server.tls = tls_resource
+        end
+        kubeclient.update_gateway(gateway_resource)
+      end
+
+      def configure_passthrough
+        set_gateway_wildcard_https(PASSTHROUGH_RESOURCE)
+      end
+
+      def gateway
+        kubeclient.get_gateway('knative-ingress-gateway', Clusters::Kubernetes::KNATIVE_SERVING_NAMESPACE)
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -231,6 +231,12 @@
   :latency_sensitive: 
   :resource_boundary: :unknown
   :weight: 1
+- :name: gcp_cluster:cluster_configure_istio
+  :feature_category: :kubernetes_management
+  :has_external_dependencies: true
+  :latency_sensitive: 
+  :resource_boundary: :unknown
+  :weight: 1
 - :name: gcp_cluster:cluster_install_app
   :feature_category: :kubernetes_management
   :has_external_dependencies: true

app/workers/cluster_configure_istio_worker.rb

+# frozen_string_literal: true
+
+class ClusterConfigureIstioWorker
+  include ApplicationWorker
+  include ClusterQueue
+
+  worker_has_external_dependencies!
+
+  def perform(cluster_id)
+    Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
+      Clusters::Kubernetes::ConfigureIstioIngressService.new(cluster: cluster).execute
+    end
+  end
+end

db/migrate/20200130134335_add_cert_and_key_to_serverless_domain_cluster.rb

+# frozen_string_literal: true
+
+class AddCertAndKeyToServerlessDomainCluster < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :serverless_domain_cluster, :encrypted_key, :text
+    add_column :serverless_domain_cluster, :encrypted_key_iv, :string, limit: 255
+    add_column :serverless_domain_cluster, :certificate, :text
+  end
+end

db/schema.rb

@@ -3786,6 +3786,9 @@ ActiveRecord::Schema.define(version: 2020_02_04_131054) do
     t.bigint "creator_id"
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
+    t.text "encrypted_key"
+    t.string "encrypted_key_iv", limit: 255
+    t.text "certificate"
     t.index ["clusters_applications_knative_id"], name: "idx_serverless_domain_cluster_on_clusters_applications_knative", unique: true
     t.index ["creator_id"], name: "index_serverless_domain_cluster_on_creator_id"
     t.index ["pages_domain_id"], name: "index_serverless_domain_cluster_on_pages_domain_id"

lib/gitlab/kubernetes/generic_secret.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class GenericSecret
+      attr_reader :name, :data, :namespace_name
+
+      def initialize(name, data, namespace_name)
+        @name = name
+        @data = data
+        @namespace_name = namespace_name
+      end
+
+      def generate
+        ::Kubeclient::Resource.new(
+          type: generic_secret_type,
+          metadata: metadata,
+          data: data
+        )
+      end
+
+      private
+
+      def generic_secret_type
+        'Opaque'
+      end
+
+      def metadata
+        {
+          name: name,
+          namespace: namespace_name
+        }
+      end
+    end
+  end
+end

lib/gitlab/kubernetes/tls_secret.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class TlsSecret
+      attr_reader :name, :cert, :key, :namespace_name
+
+      def initialize(name, cert, key, namespace_name)
+        @name = name
+        @cert = cert
+        @key = key
+        @namespace_name = namespace_name
+      end
+
+      def generate
+        ::Kubeclient::Resource.new(
+          type: tls_secret_type,
+          metadata: metadata,
+          data: data
+        )
+      end
+
+      private
+
+      def tls_secret_type
+        'kubernetes.io/tls'
+      end
+
+      def metadata
+        {
+          name: name,
+          namespace: namespace_name
+        }
+      end
+
+      def data
+        {
+          'tls.crt': Base64.strict_encode64(cert),
+          'tls.key': Base64.strict_encode64(key)
+        }
+      end
+    end
+  end
+end

spec/factories/serverless/domain_cluster.rb

@@ -5,5 +5,41 @@ FactoryBot.define do
     pages_domain { create(:pages_domain) }
     knative { create(:clusters_applications_knative) }
     creator { create(:user) }
+
+    certificate do
+      '-----BEGIN CERTIFICATE-----
+MIICGzCCAYSgAwIBAgIBATANBgkqhkiG9w0BAQUFADAbMRkwFwYDVQQDExB0ZXN0
+LWNlcnRpZmljYXRlMB4XDTE2MDIxMjE0MzIwMFoXDTIwMDQxMjE0MzIwMFowGzEZ
+MBcGA1UEAxMQdGVzdC1jZXJ0aWZpY2F0ZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEApL4J9L0ZxFJ1hI1LPIflAlAGvm6ZEvoT4qKU5Xf2JgU7/2geNR1qlNFa
+SvCc08Knupp5yTgmvyK/Xi09U0N82vvp4Zvr/diSc4A/RA6Mta6egLySNT438kdT
+nY2tR5feoTLwQpX0t4IMlwGQGT5h6Of2fKmDxzuwuyffcIHqLdsCAwEAAaNvMG0w
+DAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUxl9WSxBprB0z0ibJs3rXEk0+95AwCwYD
+VR0PBAQDAgXgMBEGCWCGSAGG+EIBAQQEAwIGQDAeBglghkgBhvhCAQ0EERYPeGNh
+IGNlcnRpZmljYXRlMA0GCSqGSIb3DQEBBQUAA4GBAGC4T8SlFHK0yPSa+idGLQFQ
+joZp2JHYvNlTPkRJ/J4TcXxBTJmArcQgTIuNoBtC+0A/SwdK4MfTCUY4vNWNdese
+5A4K65Nb7Oh1AdQieTBHNXXCdyFsva9/ScfQGEl7p55a52jOPs0StPd7g64uvjlg
+YHi2yesCrOvVXt+lgPTd
+-----END CERTIFICATE-----'
+    end
+
+    key do
+      '-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKS+CfS9GcRSdYSN
+SzyH5QJQBr5umRL6E+KilOV39iYFO/9oHjUdapTRWkrwnNPCp7qaeck4Jr8iv14t
+PVNDfNr76eGb6/3YknOAP0QOjLWunoC8kjU+N/JHU52NrUeX3qEy8EKV9LeCDJcB
+kBk+Yejn9nypg8c7sLsn33CB6i3bAgMBAAECgYA2D26w80T7WZvazYr86BNMePpd
+j2mIAqx32KZHzt/lhh40J/SRtX9+Kl0Y7nBoRR5Ja9u/HkAIxNxLiUjwg9r6cpg/
+uITEF5nMt7lAk391BuI+7VOZZGbJDsq2ulPd6lO+C8Kq/PI/e4kXcIjeH6KwQsuR
+5vrXfBZ3sQfflaiN4QJBANBt8JY2LIGQF8o89qwUpRL5vbnKQ4IzZ5+TOl4RLR7O
+AQpJ81tGuINghO7aunctb6rrcKJrxmEH1whzComybrMCQQDKV49nOBudRBAIgG4K
+EnLzsRKISUHMZSJiYTYnablof8cKw1JaQduw7zgrUlLwnroSaAGX88+Jw1f5n2Lh
+Vlg5AkBDdUGnrDLtYBCDEQYZHblrkc7ZAeCllDOWjxUV+uMqlCv8A4Ey6omvY57C
+m6I8DkWVAQx8VPtozhvHjUw80rZHAkB55HWHAM3h13axKG0htCt7klhPsZHpx6MH
+EPjGlXIT+aW2XiPmK3ZlCDcWIenE+lmtbOpI159Wpk8BGXs/s/xBAkEAlAY3ymgx
+63BDJEwvOb2IaP8lDDxNsXx9XJNVvQbv5n15vNsLHbjslHfAhAbxnLQ1fLhUPqSi
+nNp/xedE1YxutQ==
+-----END PRIVATE KEY-----'
+    end
   end
 end

spec/lib/gitlab/kubernetes/generic_secret_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Kubernetes::GenericSecret do
+  let(:secret) { described_class.new(name, data, namespace) }
+  let(:name) { 'example-name' }
+  let(:data) { 'example-data' }
+  let(:namespace) { 'example-namespace' }
+
+  describe '#generate' do
+    subject { secret.generate }
+
+    let(:resource) do
+      ::Kubeclient::Resource.new(
+        type: 'Opaque',
+        metadata: { name: name, namespace: namespace },
+        data: data
+      )
+    end
+
+    it { is_expected.to eq(resource) }
+  end
+end

spec/lib/gitlab/kubernetes/tls_secret_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Kubernetes::TlsSecret do
+  let(:secret) { described_class.new(name, cert, key, namespace) }
+  let(:name) { 'example-name' }
+  let(:cert) { 'example-cert' }
+  let(:key) { 'example-key' }
+  let(:namespace) { 'example-namespace' }
+
+  let(:data) do
+    {
+      'tls.crt': Base64.strict_encode64(cert),
+      'tls.key': Base64.strict_encode64(key)
+    }
+  end
+
+  describe '#generate' do
+    subject { secret.generate }
+
+    let(:resource) do
+      ::Kubeclient::Resource.new(
+        type: 'kubernetes.io/tls',
+        metadata: { name: name, namespace: namespace },
+        data: data
+      )
+    end
+
+    it { is_expected.to eq(resource) }
+  end
+end

spec/models/serverless/domain_cluster_spec.rb

@@ -50,4 +50,12 @@ describe ::Serverless::DomainCluster do
   describe 'domain' do
     it { is_expected.to respond_to(:domain) }
   end
+
+  describe 'certificate' do
+    it { is_expected.to respond_to(:certificate) }
+  end
+
+  describe 'key' do
+    it { is_expected.to respond_to(:key) }
+  end
 end

spec/services/clusters/kubernetes/configure_istio_ingress_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Kubernetes::ConfigureIstioIngressService, '#execute' do
+  include KubernetesHelpers
+
+  let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
+  let(:api_url) { 'https://kubernetes.example.com' }
+  let(:project) { cluster.project }
+  let(:environment) { create(:environment, project: project) }
+  let(:cluster_project) { cluster.cluster_project }
+  let(:namespace) { "#{project.name}-#{project.id}-#{environment.slug}" }
+  let(:kubeclient) { cluster.kubeclient }
+
+  subject do
+    described_class.new(
+      cluster: cluster
+    ).execute
+  end
+
+  before do
+    stub_kubeclient_discover_istio(api_url)
+    stub_kubeclient_create_secret(api_url, namespace: namespace)
+    stub_kubeclient_put_secret(api_url, "#{namespace}-token", namespace: namespace)
+
+    stub_kubeclient_get_secret(
+      api_url,
+      {
+        metadata_name: "#{namespace}-token",
+        token: Base64.encode64('sample-token'),
+        namespace: namespace
+      }
+    )
+
+    stub_kubeclient_get_secret(
+      api_url,
+      {
+        metadata_name: 'istio-ingressgateway-ca-certs',
+        namespace: 'istio-system'
+      }
+    )
+
+    stub_kubeclient_get_secret(
+      api_url,
+      {
+        metadata_name: 'istio-ingressgateway-certs',
+        namespace: 'istio-system'
+      }
+    )
+
+    stub_kubeclient_put_secret(api_url, 'istio-ingressgateway-ca-certs', namespace: 'istio-system')
+    stub_kubeclient_put_secret(api_url, 'istio-ingressgateway-certs', namespace: 'istio-system')
+    stub_kubeclient_get_gateway(api_url, 'knative-ingress-gateway', namespace: 'knative-serving')
+    stub_kubeclient_put_gateway(api_url, 'knative-ingress-gateway', namespace: 'knative-serving')
+  end
+
+  context 'without a serverless_domain_cluster' do
+    it 'configures gateway to use PASSTHROUGH' do
+      subject
+
+      expect(WebMock).to have_requested(:put, api_url + '/apis/networking.istio.io/v1alpha3/namespaces/knative-serving/gateways/knative-ingress-gateway').with(
+        body: hash_including(
+          apiVersion: "networking.istio.io/v1alpha3",
+          kind: "Gateway",
+          metadata: {
+            generation: 1,
+            labels: {
+              "networking.knative.dev/ingress-provider" => "istio",
+              "serving.knative.dev/release" => "v0.7.0"
+            },
+            name: "knative-ingress-gateway",
+            namespace: "knative-serving",
+            selfLink: "/apis/networking.istio.io/v1alpha3/namespaces/knative-serving/gateways/knative-ingress-gateway"
+          },
+          spec: {
+            selector: {
+              istio: "ingressgateway"
+            },
+            servers: [
+              {
+                hosts: ["*"],
+                port: {
+                  name: "http",
+                  number: 80,
+                  protocol: "HTTP"
+                }
+              },
+              {
+                hosts: ["*"],
+                port: {
+                  name: "https",
+                  number: 443,
+                  protocol: "HTTPS"
+                },
+                tls: {
+                  mode: "PASSTHROUGH"
+                }
+              }
+            ]
+          }
+        )
+      )
+    end
+  end
+
+  context 'with a serverless_domain_cluster' do
+    let(:serverless_domain_cluster) { create(:serverless_domain_cluster) }
+    let(:certificate) { OpenSSL::X509::Certificate.new(serverless_domain_cluster.certificate) }
+
+    before do
+      cluster.application_knative = serverless_domain_cluster.knative
+    end
+
+    it 'configures certificates' do
+      subject
+
+      expect(serverless_domain_cluster.reload.key).not_to be_blank
+      expect(serverless_domain_cluster.reload.certificate).not_to be_blank
+
+      expect(certificate.subject.to_s).to include(serverless_domain_cluster.knative.hostname)
+
+      expect(certificate.not_before).to be_within(1.minute).of(Time.now)
+      expect(certificate.not_after).to be_within(1.minute).of(Time.now + 1000.years)
+
+      expect(WebMock).to have_requested(:put, api_url + '/api/v1/namespaces/istio-system/secrets/istio-ingressgateway-ca-certs').with(
+        body: hash_including(
+          metadata: {
+            name: 'istio-ingressgateway-ca-certs',
+            namespace: 'istio-system'
+          },
+          type: 'Opaque'
+        )
+      )
+
+      expect(WebMock).to have_requested(:put, api_url + '/api/v1/namespaces/istio-system/secrets/istio-ingressgateway-certs').with(
+        body: hash_including(
+          metadata: {
+            name: 'istio-ingressgateway-certs',
+            namespace: 'istio-system'
+          },
+          type: 'kubernetes.io/tls'
+        )
+      )
+    end
+
+    it 'configures gateway to use MUTUAL' do
+      subject
+
+      expect(WebMock).to have_requested(:put, api_url + '/apis/networking.istio.io/v1alpha3/namespaces/knative-serving/gateways/knative-ingress-gateway').with(
+        body: {
+          apiVersion: "networking.istio.io/v1alpha3",
+          kind: "Gateway",
+          metadata: {
+            generation: 1,
+            labels: {
+              "networking.knative.dev/ingress-provider" => "istio",
+              "serving.knative.dev/release" => "v0.7.0"
+            },
+            name: "knative-ingress-gateway",
+            namespace: "knative-serving",
+            selfLink: "/apis/networking.istio.io/v1alpha3/namespaces/knative-serving/gateways/knative-ingress-gateway"
+          },
+          spec: {
+            selector: {
+              istio: "ingressgateway"
+            },
+            servers: [
+              {
+                hosts: ["*"],
+                port: {
+                  name: "http",
+                  number: 80,
+                  protocol: "HTTP"
+                }
+              },
+              {
+                hosts: ["*"],
+                port: {
+                  name: "https",
+                  number: 443,
+                  protocol: "HTTPS"
+                },
+                tls: {
+                  mode: "MUTUAL",
+                  privateKey: "/etc/istio/ingressgateway-certs/tls.key",
+                  serverCertificate: "/etc/istio/ingressgateway-certs/tls.crt",
+                  caCertificates: "/etc/istio/ingressgateway-ca-certs/cert.pem"
+                }
+              }
+            ]
+          }
+        }
+      )
+    end
+  end
+end

spec/workers/cluster_configure_istio_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ClusterConfigureIstioWorker do
+  describe '#perform' do
+    shared_examples 'configure istio service' do
+      it 'configures istio' do
+        expect_any_instance_of(Clusters::Kubernetes::ConfigureIstioIngressService).to receive(:execute)
+
+        described_class.new.perform(cluster.id)
+      end
+    end
+
+    context 'when provider type is gcp' do
+      let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
+
+      it_behaves_like 'configure istio service'
+    end
+
+    context 'when provider type is aws' do
+      let(:cluster) { create(:cluster, :project, :provided_by_aws) }
+
+      it_behaves_like 'configure istio service'
+    end
+
+    context 'when provider type is user' do
+      let(:cluster) { create(:cluster, :project, :provided_by_user) }
+
+      it_behaves_like 'configure istio service'
+    end
+
+    context 'when cluster does not exist' do
+      it 'does not provision a cluster' do
+        expect_any_instance_of(Clusters::Kubernetes::ConfigureIstioIngressService).not_to receive(:execute)
+
+        described_class.new.perform(123)
+      end
+    end
+  end
+end