Introduce Canary Object Model to check the canary status

This commit introduces the Canary Object Model

app/serializers/deployment_entity.rb

@@ -17,6 +17,7 @@ class DeploymentEntity < Grape::Entity
     end
   end
 
+  expose :status
   expose :created_at
   expose :deployed_at
   expose :tag

config/feature_flags/development/canary_ingress_weight_control.yml

+---
+name: canary_ingress_weight_control
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43816
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/260295
+type: development
+group: group::progressive delivery
+default_enabled: false

ee/app/models/ee/clusters/platforms/kubernetes.rb

@@ -14,8 +14,15 @@ module EE
           if result
             deployments = read_deployments(environment.deployment_namespace)
 
+            ingresses = if ::Feature.enabled?(:canary_ingress_weight_control, environment.project)
+                          read_ingresses(environment.deployment_namespace)
+                        else
+                          []
+                        end
+
             # extract_relevant_deployment_data avoids uploading all the deployment info into ReactiveCaching
             result[:deployments] = extract_relevant_deployment_data(deployments)
+            result[:ingresses] = extract_relevant_ingress_data(ingresses)
           end
 
           result
@@ -26,8 +33,9 @@ module EE
 
           deployments = filter_by_project_environment(data[:deployments], project.full_path_slug, environment.slug)
           pods = filter_by_project_environment(data[:pods], project.full_path_slug, environment.slug)
+          ingresses = data[:ingresses].presence || []
 
-          ::Gitlab::Kubernetes::RolloutStatus.from_deployments(*deployments, pods_attrs: pods)
+          ::Gitlab::Kubernetes::RolloutStatus.from_deployments(*deployments, pods_attrs: pods, ingresses: ingresses)
         end
 
         private
@@ -38,6 +46,12 @@ module EE
           []
         end
 
+        def read_ingresses(namespace)
+          kubeclient.get_ingresses(namespace: namespace).as_json
+        rescue Kubeclient::ResourceNotFoundError
+          []
+        end
+
         def extract_relevant_deployment_data(deployments)
           deployments.map do |deployment|
             {
@@ -47,6 +61,14 @@ module EE
             }
           end
         end
+
+        def extract_relevant_ingress_data(ingresses)
+          ingresses.map do |ingress|
+            {
+              'metadata' => ingress.fetch('metadata', {}).slice('name', 'labels', 'annotations')
+            }
+          end
+        end
       end
     end
   end

ee/app/serializers/rollout_status_entity.rb

@@ -13,4 +13,6 @@ class RolloutStatusEntity < Grape::Entity
   expose :instances, if: -> (rollout_status, _) { rollout_status.found? }
   expose :completion, if: -> (rollout_status, _) { rollout_status.found? }
   expose :complete?, as: :is_completed, if: -> (rollout_status, _) { rollout_status.found? }
+  expose :canary_ingress, using: RolloutStatuses::IngressEntity, expose_nil: false,
+         if: -> (rollout_status, _) { rollout_status.found? && rollout_status.canary_ingress_exists? }
 end

ee/app/serializers/rollout_statuses/ingress_entity.rb

+# frozen_string_literal: true
+
+module RolloutStatuses
+  class IngressEntity < Grape::Entity
+    expose :canary_weight
+  end
+end

ee/lib/gitlab/kubernetes/ingress.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class Ingress
+      include Gitlab::Utils::StrongMemoize
+
+      # Canry Ingress Annotations https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary
+      ANNOTATION_KEY_CANARY = 'nginx.ingress.kubernetes.io/canary'
+      ANNOTATION_KEY_CANARY_WEIGHT = 'nginx.ingress.kubernetes.io/canary-weight'
+
+      def initialize(attributes = {})
+        @attributes = attributes
+      end
+
+      def canary?
+        strong_memoize(:is_canary) do
+          annotations.any? do |key, value|
+            key == ANNOTATION_KEY_CANARY && value == 'true'
+          end
+        end
+      end
+
+      def canary_weight
+        return unless canary?
+        return unless annotations.key?(ANNOTATION_KEY_CANARY_WEIGHT)
+
+        annotations[ANNOTATION_KEY_CANARY_WEIGHT].to_i
+      end
+
+      private
+
+      def metadata
+        @attributes.fetch('metadata', {})
+      end
+
+      def annotations
+        metadata.fetch('annotations', {})
+      end
+    end
+  end
+end

ee/lib/gitlab/kubernetes/rollout_status.rb

@@ -8,7 +8,7 @@ module Gitlab
     # other resources. The rollout status sums the Kubernetes deployments
     # together.
     class RolloutStatus
-      attr_reader :deployments, :instances, :completion, :status
+      attr_reader :deployments, :instances, :completion, :status, :canary_ingress
 
       def complete?
         completion == 100
@@ -26,7 +26,11 @@ module Gitlab
         @status == :found
       end
 
-      def self.from_deployments(*deployments_attrs, pods_attrs: [])
+      def canary_ingress_exists?
+        canary_ingress.present?
+      end
+
+      def self.from_deployments(*deployments_attrs, pods_attrs: [], ingresses: [])
         return new([], status: :not_found) if deployments_attrs.empty?
 
         deployments = deployments_attrs.map do |attrs|
@@ -38,14 +42,16 @@ module Gitlab
           ::Gitlab::Kubernetes::Pod.new(attrs)
         end
 
-        new(deployments, pods: pods)
+        ingresses = ingresses.map { |ingress| ::Gitlab::Kubernetes::Ingress.new(ingress) }
+
+        new(deployments, pods: pods, ingresses: ingresses)
       end
 
       def self.loading
         new([], status: :loading)
       end
 
-      def initialize(deployments, pods: [], status: :found)
+      def initialize(deployments, pods: [], ingresses: [], status: :found)
         @status       = status
         @deployments  = deployments
 
@@ -55,6 +61,8 @@ module Gitlab
                        deployments.flat_map(&:instances)
                      end
 
+        @canary_ingress = ingresses.find(&:canary?)
+
         @completion =
           if @instances.empty?
             100

ee/spec/lib/gitlab/kubernetes/ingress_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kubernetes::Ingress do
+  include KubernetesHelpers
+
+  let(:ingress) { described_class.new(params) }
+
+  describe '#canary?' do
+    subject { ingress.canary? }
+
+    context 'with canary ingress parameters' do
+      let(:params) { canary_metadata }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'with stable ingress parameters' do
+      let(:params) { stable_metadata }
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#canary_weight' do
+    subject { ingress.canary_weight }
+
+    context 'with canary ingress parameters' do
+      let(:params) { canary_metadata }
+
+      it { is_expected.to eq(50) }
+    end
+
+    context 'with stable ingress parameters' do
+      let(:params) { stable_metadata }
+
+      it { is_expected.to be_nil }
+    end
+  end
+
+  def stable_metadata
+    kube_ingress(track: :stable)
+  end
+
+  def canary_metadata
+    kube_ingress(track: :canary)
+  end
+end

ee/spec/lib/gitlab/kubernetes/rollout_status_spec.rb

@@ -12,6 +12,8 @@ RSpec.describe Gitlab::Kubernetes::RolloutStatus do
     create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: "canary")
   end
 
+  let(:ingresses) { [] }
+
   let(:specs_all_finished) do
     [
       kube_deployment(name: 'one'),
@@ -26,7 +28,7 @@ RSpec.describe Gitlab::Kubernetes::RolloutStatus do
     ]
   end
 
-  subject(:rollout_status) { described_class.from_deployments(*specs, pods_attrs: pods) }
+  subject(:rollout_status) { described_class.from_deployments(*specs, pods_attrs: pods, ingresses: ingresses) }
 
   shared_examples 'rollout status' do
     describe '#deployments' do
@@ -223,6 +225,24 @@ RSpec.describe Gitlab::Kubernetes::RolloutStatus do
         it { is_expected.to be_not_found }
       end
     end
+
+    describe '#canary_ingress_exists?' do
+      context 'when canary ingress exists' do
+        let(:ingresses) { [kube_ingress(track: :canary)] }
+
+        it 'returns true' do
+          expect(rollout_status.canary_ingress_exists?).to eq(true)
+        end
+      end
+
+      context 'when canary ingress does not exist' do
+        let(:ingresses) { [kube_ingress(track: :stable)] }
+
+        it 'returns false' do
+          expect(rollout_status.canary_ingress_exists?).to eq(false)
+        end
+      end
+    end
   end
 
   context 'deploy_boards_dedupe_instances is disabled' do

ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb

@@ -27,11 +27,12 @@ RSpec.describe Clusters::Platforms::Kubernetes do
   describe '#rollout_status' do
     let(:deployments) { [] }
     let(:pods) { [] }
+    let(:ingresses) { [] }
     let(:service) { create(:cluster_platform_kubernetes, :configured) }
     let!(:cluster) { create(:cluster, :project, enabled: true, platform_kubernetes: service) }
     let(:project) { cluster.project }
     let(:environment) { build(:environment, project: project, name: "env", slug: "env-000000") }
-    let(:cache_data) { Hash(deployments: deployments, pods: pods) }
+    let(:cache_data) { Hash(deployments: deployments, pods: pods, ingresses: ingresses) }
 
     subject(:rollout_status) { service.rollout_status(environment, cache_data) }
 
@@ -129,6 +130,15 @@ RSpec.describe Clusters::Platforms::Kubernetes do
                                                  tooltip: "Not provided (Pending)",
                                                  track: "stable" }])
       end
+
+      context 'with canary ingress' do
+        let(:ingresses) { [kube_ingress(track: :canary)] }
+
+        it 'has canary ingress' do
+          expect(rollout_status).to be_canary_ingress_exists
+          expect(rollout_status.canary_ingress.canary_weight).to eq(50)
+        end
+      end
     end
 
     context 'with empty list of deployments' do
@@ -304,7 +314,7 @@ RSpec.describe Clusters::Platforms::Kubernetes do
     let(:cluster) { create(:cluster, :project, platform_kubernetes: service) }
     let(:service) { create(:cluster_platform_kubernetes, :configured) }
     let(:namespace) { 'project-namespace' }
-    let(:environment) { instance_double(Environment, deployment_namespace: namespace) }
+    let(:environment) { instance_double(Environment, deployment_namespace: namespace, project: cluster.project) }
     let(:expected_pod_cached_data) do
       kube_pod.tap { |kp| kp['metadata'].delete('namespace') }
     end
@@ -315,10 +325,11 @@ RSpec.describe Clusters::Platforms::Kubernetes do
       before do
         stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace)
+        stub_kubeclient_ingresses(namespace)
       end
 
       shared_examples 'successful deployment request' do
-        it { is_expected.to include(pods: [expected_pod_cached_data], deployments: [kube_deployment]) }
+        it { is_expected.to include(pods: [expected_pod_cached_data], deployments: [kube_deployment], ingresses: [kube_ingress]) }
       end
 
       context 'on a project level cluster' do
@@ -338,6 +349,16 @@ RSpec.describe Clusters::Platforms::Kubernetes do
 
         include_examples 'successful deployment request'
       end
+
+      context 'when canary_ingress_weight_control feature flag is disabled' do
+        before do
+          stub_feature_flags(canary_ingress_weight_control: false)
+        end
+
+        it 'does not fetch ingress data from kubernetes' do
+          expect(subject[:ingresses]).to be_empty
+        end
+      end
     end
 
     context 'when kubernetes responds with 500s' do
@@ -353,9 +374,10 @@ RSpec.describe Clusters::Platforms::Kubernetes do
       before do
         stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace, status: 404)
+        stub_kubeclient_ingresses(namespace, status: 404)
       end
 
-      it { is_expected.to include(deployments: []) }
+      it { is_expected.to include(deployments: [], ingresses: []) }
     end
   end
 end

ee/spec/serializers/rollout_status_entity_spec.rb

@@ -25,6 +25,18 @@ RSpec.describe RolloutStatusEntity do
     it "exposes deployment data" do
       is_expected.to include(:instances, :completion, :is_completed)
     end
+
+    it 'does not expose canary ingress if it does not exist' do
+      is_expected.not_to include(:canary_ingress)
+    end
+
+    context 'when canary ingress exists' do
+      let(:rollout_status) { kube_deployment_rollout_status(ingresses: [kube_ingress(track: :canary)]) }
+
+      it 'expose canary ingress' do
+        is_expected.to include(:canary_ingress)
+      end
+    end
   end
 
   context 'when kube deployment is empty' do
@@ -35,7 +47,7 @@ RSpec.describe RolloutStatusEntity do
     end
 
     it "does not expose deployment data" do
-      is_expected.not_to include(:instances, :completion, :is_completed)
+      is_expected.not_to include(:instances, :completion, :is_completed, :canary_ingress)
     end
   end
 end

ee/spec/serializers/rollout_statuses/ingress_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RolloutStatuses::IngressEntity do
+  include KubernetesHelpers
+
+  let(:canary_ingress) { kube_ingress(track: :canary) }
+
+  let(:entity) do
+    described_class.new(canary_ingress, request: double)
+  end
+
+  subject { entity.as_json }
+
+  it 'exposes canary weight' do
+    is_expected.to include(:canary_weight)
+  end
+end

lib/gitlab/kubernetes/kube_client.rb

@@ -167,6 +167,21 @@ module Gitlab
         end
       end
 
+      # Ingresses resource is currently on the apis/extensions api group
+      # until Kubernetes 1.21. Kubernetest 1.22+ has ingresses resources in
+      # the networking.k8s.io/v1 api group.
+      #
+      # As we still support Kubernetes 1.12+, we will need to support both.
+      def get_ingresses(**args)
+        extensions_client.discover unless extensions_client.discovered
+
+        if extensions_client.respond_to?(:get_ingresses)
+          extensions_client.get_ingresses(**args)
+        else
+          networking_client.get_ingresses(**args)
+        end
+      end
+
       def create_or_update_cluster_role_binding(resource)
         update_cluster_role_binding(resource)
       end

spec/lib/gitlab/kubernetes/kube_client_spec.rb

@@ -347,6 +347,34 @@ RSpec.describe Gitlab::Kubernetes::KubeClient do
     end
   end
 
+  describe '#get_ingresses' do
+    let(:extensions_client) { client.extensions_client }
+    let(:networking_client) { client.networking_client }
+
+    include_examples 'redirection not allowed', 'get_ingresses'
+    include_examples 'dns rebinding not allowed', 'get_ingresses'
+
+    it 'delegates to the extensions client' do
+      expect(extensions_client).to receive(:get_ingresses)
+
+      client.get_ingresses
+    end
+
+    context 'extensions does not have deployments for Kubernetes 1.22+ clusters' do
+      before do
+        WebMock
+          .stub_request(:get, api_url + '/apis/extensions/v1beta1')
+          .to_return(kube_response(kube_1_22_extensions_v1beta1_discovery_body))
+      end
+
+      it 'delegates to the apps client' do
+        expect(networking_client).to receive(:get_ingresses)
+
+        client.get_ingresses
+      end
+    end
+  end
+
   describe 'istio API group' do
     let(:istio_client) { client.istio_client }
 

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -412,7 +412,7 @@ RSpec.describe Clusters::Platforms::Kubernetes do
     end
 
     let(:namespace) { "project-namespace" }
-    let(:environment) { instance_double(Environment, deployment_namespace: namespace) }
+    let(:environment) { instance_double(Environment, deployment_namespace: namespace, project: service.cluster.project) }
 
     subject { service.calculate_reactive_cache_for(environment) }
 
@@ -428,6 +428,7 @@ RSpec.describe Clusters::Platforms::Kubernetes do
       before do
         stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace)
+        stub_kubeclient_ingresses(namespace)
       end
 
       it { is_expected.to include(pods: [expected_pod_cached_data]) }
@@ -437,6 +438,7 @@ RSpec.describe Clusters::Platforms::Kubernetes do
       before do
         stub_kubeclient_pods(namespace, status: 500)
         stub_kubeclient_deployments(namespace, status: 500)
+        stub_kubeclient_ingresses(namespace, status: 500)
       end
 
       it { expect { subject }.to raise_error(Kubeclient::HttpError) }
@@ -446,6 +448,7 @@ RSpec.describe Clusters::Platforms::Kubernetes do
       before do
         stub_kubeclient_pods(namespace, status: 404)
         stub_kubeclient_deployments(namespace, status: 404)
+        stub_kubeclient_ingresses(namespace, status: 404)
       end
 
       it { is_expected.to include(pods: []) }

spec/serializers/deployment_entity_spec.rb

@@ -30,6 +30,10 @@ RSpec.describe DeploymentEntity do
     expect(subject[:ref][:name]).to eq 'master'
   end
 
+  it 'exposes status' do
+    expect(subject).to include(:status)
+  end
+
   it 'exposes creation date' do
     expect(subject).to include(:created_at)
   end

spec/support/helpers/kubernetes_helpers.rb

@@ -33,6 +33,10 @@ module KubernetesHelpers
     kube_response(kube_deployments_body)
   end
 
+  def kube_ingresses_response
+    kube_response(kube_ingresses_body)
+  end
+
   def stub_kubeclient_discover_base(api_url)
     WebMock.stub_request(:get, api_url + '/api/v1').to_return(kube_response(kube_v1_discovery_body))
     WebMock
@@ -63,6 +67,9 @@ module KubernetesHelpers
     WebMock
       .stub_request(:get, api_url + '/apis/serving.knative.dev/v1alpha1')
       .to_return(kube_response(kube_v1alpha1_serving_knative_discovery_body))
+    WebMock
+      .stub_request(:get, api_url + '/apis/networking.k8s.io/v1')
+      .to_return(kube_response(kube_v1_networking_discovery_body))
   end
 
   def stub_kubeclient_discover_knative_not_found(api_url)
@@ -148,6 +155,14 @@ module KubernetesHelpers
     WebMock.stub_request(:get, deployments_url).to_return(response || kube_deployments_response)
   end
 
+  def stub_kubeclient_ingresses(namespace, status: nil)
+    stub_kubeclient_discover(service.api_url)
+    ingresses_url = service.api_url + "/apis/extensions/v1beta1/namespaces/#{namespace}/ingresses"
+    response = { status: status } if status
+
+    WebMock.stub_request(:get, ingresses_url).to_return(response || kube_ingresses_response)
+  end
+
   def stub_kubeclient_knative_services(options = {})
     namespace_path = options[:namespace].present? ? "namespaces/#{options[:namespace]}/" : ""
 
@@ -304,6 +319,14 @@ module KubernetesHelpers
     }
   end
 
+  # From Kubernetes 1.22+ Ingresses are no longer served from apis/extensions
+  def kube_1_22_extensions_v1beta1_discovery_body
+    {
+      "kind" => "APIResourceList",
+      "resources" => []
+    }
+  end
+
   def kube_knative_discovery_body
     {
       "kind" => "APIResourceList",
@@ -416,6 +439,17 @@ module KubernetesHelpers
     }
   end
 
+  def kube_v1_networking_discovery_body
+    {
+      "kind" => "APIResourceList",
+      "apiVersion" => "v1",
+      "groupVersion" => "networking.k8s.io/v1",
+      "resources" => [
+        { "name" => "ingresses", "namespaced" => true, "kind" => "Ingress" }
+      ]
+    }
+  end
+
   def kube_istio_gateway_body(name, namespace)
     {
       "apiVersion" => "networking.istio.io/v1alpha3",
@@ -507,6 +541,13 @@ module KubernetesHelpers
     }
   end
 
+  def kube_ingresses_body
+    {
+      "kind" => "List",
+      "items" => [kube_ingress]
+    }
+  end
+
   def kube_knative_pods_body(name, namespace)
     {
       "kind" => "PodList",
@@ -548,6 +589,38 @@ module KubernetesHelpers
     }
   end
 
+  def kube_ingress(track: :stable)
+    additional_annotations =
+      if track == :canary
+        {
+          "nginx.ingress.kubernetes.io/canary" => "true",
+          "nginx.ingress.kubernetes.io/canary-by-header" => "canary",
+          "nginx.ingress.kubernetes.io/canary-weight" => "50"
+        }
+      else
+        {}
+      end
+
+    {
+      "metadata" => {
+        "name" => "production-auto-deploy",
+        "labels" => {
+          "app" => "production",
+          "app.kubernetes.io/managed-by" => "Helm",
+          "chart" => "auto-deploy-app-2.0.0-beta.2",
+          "heritage" => "Helm",
+          "release" => "production"
+        },
+        "annotations" => {
+          "kubernetes.io/ingress.class" => "nginx",
+          "kubernetes.io/tls-acme" => "true",
+          "meta.helm.sh/release-name" => "production",
+          "meta.helm.sh/release-namespace" => "awesome-app-1-production"
+        }.merge(additional_annotations)
+      }
+    }
+  end
+
   # This is a partial response, it will have many more elements in reality but
   # these are the ones we care about at the moment
   def kube_node
@@ -862,8 +935,8 @@ module KubernetesHelpers
     end
   end
 
-  def kube_deployment_rollout_status
-    ::Gitlab::Kubernetes::RolloutStatus.from_deployments(kube_deployment)
+  def kube_deployment_rollout_status(ingresses: [])
+    ::Gitlab::Kubernetes::RolloutStatus.from_deployments(kube_deployment, ingresses: ingresses)
   end
 
   def empty_deployment_rollout_status