Commit a1b56a54 authored by Jason Goodman's avatar Jason Goodman Committed by Thong Kuah

Return each pod only once for deploy board with multiple deployments

Fixes an issue where pods were returned multiple times
parent c94aa77f
---
name: deploy_boards_dedupe_instances
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40768
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/258214
type: development
group: group::progressive-delivery
default_enabled: false
......@@ -42,8 +42,9 @@ knowledge. In particular, you should be familiar with:
- [Kubernetes canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments)
NOTE: **Note:**
Apps that consist of multiple deployments are shown as duplicates on the deploy board.
Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/8463) for details.
In GitLab 13.4 and earlier, apps that consist of multiple deployments are shown as
duplicates on the deploy board. This is [fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/8463)
in GitLab 13.5.
## Use cases
......
---
title: Fix duplicate instances in deploy boards when multiple deployments have the same track
merge_request: 40768
author:
type: fixed
# frozen_string_literal: true
module Gitlab
module Kubernetes
class RolloutInstances
include ::Gitlab::Utils::StrongMemoize
def initialize(deployments, pods)
@deployments = deployments
@pods = pods
end
def pod_instances
pods = matching_pods + extra_pending_pods
pods.sort_by(&:order).map do |pod|
to_hash(pod)
end
end
private
attr_reader :deployments, :pods
def matching_pods
strong_memoize(:matching_pods) do
deployment_tracks = deployments.map(&:track)
pods.select { |p| deployment_tracks.include?(p.track) }
end
end
def extra_pending_pods
wanted_instances = sum_hashes(deployments.map { |d| { d.track => d.wanted_instances } })
present_instances = sum_hashes(matching_pods.map { |p| { p.track => 1 } })
pending_instances = subtract_hashes(wanted_instances, present_instances)
pending_instances.flat_map do |track, num|
Array.new(num, pending_pod_for(track))
end
end
def sum_hashes(hashes)
hashes.reduce({}) do |memo, hash|
memo.merge(hash) { |_key, memo_val, hash_val| memo_val + hash_val }
end
end
def subtract_hashes(hash_a, hash_b)
hash_a.merge(hash_b) { |_key, val_a, val_b| [0, val_a - val_b].max }
end
def pending_pod_for(track)
::Gitlab::Kubernetes::Pod.new({
'status' => { 'phase' => 'Pending' },
'metadata' => {
'name' => 'Not provided',
'labels' => {
'track' => track
}
}
})
end
def to_hash(pod)
{
status: pod.status&.downcase,
pod_name: pod.name,
tooltip: "#{pod.name} (#{pod.status})",
track: pod.track,
stable: pod.stable?
}
end
end
end
end
......@@ -26,29 +26,41 @@ module Gitlab
@status == :found
end
def self.from_deployments(*deployments, pods_attrs: {})
return new([], status: :not_found) if deployments.empty?
def self.from_deployments(*deployments_attrs, pods_attrs: [])
return new([], status: :not_found) if deployments_attrs.empty?
deployments = deployments.map { |deploy| ::Gitlab::Kubernetes::Deployment.new(deploy, pods: pods_attrs) }
deployments = deployments_attrs.map do |attrs|
::Gitlab::Kubernetes::Deployment.new(attrs, pods: pods_attrs)
end
deployments.sort_by!(&:order)
new(deployments)
pods = pods_attrs.map do |attrs|
::Gitlab::Kubernetes::Pod.new(attrs)
end
new(deployments, pods: pods)
end
def self.loading
new([], status: :loading)
end
def initialize(deployments, status: :found)
def initialize(deployments, pods: [], status: :found)
@status = status
@deployments = deployments
@instances = deployments.flat_map(&:instances)
@instances = if ::Feature.enabled?(:deploy_boards_dedupe_instances)
RolloutInstances.new(deployments, pods).pod_instances
else
deployments.flat_map(&:instances)
end
@completion =
if @instances.empty?
100
else
# We downcase the pod status in Gitlab::Kubernetes::Deployment#deployment_instance
finished = @instances.count { |instance| instance[:status] == Gitlab::Kubernetes::Pod::RUNNING.downcase }
finished = @instances.count { |instance| instance[:status] == ::Gitlab::Kubernetes::Pod::RUNNING.downcase }
(finished / @instances.count.to_f * 100).to_i
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Kubernetes::RolloutInstances do
include KubernetesHelpers
def setup(deployments_attrs, pods_attrs)
deployments = deployments_attrs.map do |attrs|
::Gitlab::Kubernetes::Deployment.new(attrs, pods: pods_attrs)
end
pods = pods_attrs.map do |attrs|
::Gitlab::Kubernetes::Pod.new(attrs)
end
[deployments, pods]
end
describe '#pod_instances' do
it 'returns an instance for a deployment with one pod' do
deployments, pods = setup(
[kube_deployment(name: 'one', track: 'stable', replicas: 1)],
[kube_pod(name: 'one', status: 'Running', track: 'stable')]
)
rollout_instances = described_class.new(deployments, pods)
expect(rollout_instances.pod_instances).to eq([{
pod_name: 'one',
stable: true,
status: 'running',
tooltip: 'one (Running)',
track: 'stable'
}])
end
it 'returns a pending pod for a missing replica' do
deployments, pods = setup(
[kube_deployment(name: 'one', track: 'stable', replicas: 1)],
[]
)
rollout_instances = described_class.new(deployments, pods)
expect(rollout_instances.pod_instances).to eq([{
pod_name: 'Not provided',
stable: true,
status: 'pending',
tooltip: 'Not provided (Pending)',
track: 'stable'
}])
end
it 'returns instances when there are two stable deployments' do
deployments, pods = setup([
kube_deployment(name: 'one', track: 'stable', replicas: 1),
kube_deployment(name: 'two', track: 'stable', replicas: 1)
], [
kube_pod(name: 'one', status: 'Running', track: 'stable'),
kube_pod(name: 'two', status: 'Running', track: 'stable')
])
rollout_instances = described_class.new(deployments, pods)
expect(rollout_instances.pod_instances).to eq([{
pod_name: 'one',
stable: true,
status: 'running',
tooltip: 'one (Running)',
track: 'stable'
}, {
pod_name: 'two',
stable: true,
status: 'running',
tooltip: 'two (Running)',
track: 'stable'
}])
end
it 'returns instances for two deployments with different tracks' do
deployments, pods = setup([
kube_deployment(name: 'one', track: 'mytrack', replicas: 1),
kube_deployment(name: 'two', track: 'othertrack', replicas: 1)
], [
kube_pod(name: 'one', status: 'Running', track: 'mytrack'),
kube_pod(name: 'two', status: 'Running', track: 'othertrack')
])
rollout_instances = described_class.new(deployments, pods)
expect(rollout_instances.pod_instances).to eq([{
pod_name: 'one',
stable: false,
status: 'running',
tooltip: 'one (Running)',
track: 'mytrack'
}, {
pod_name: 'two',
stable: false,
status: 'running',
tooltip: 'two (Running)',
track: 'othertrack'
}])
end
it 'sorts stable tracks after canary tracks' do
deployments, pods = setup([
kube_deployment(name: 'one', track: 'stable', replicas: 1),
kube_deployment(name: 'two', track: 'canary', replicas: 1)
], [
kube_pod(name: 'one', status: 'Running', track: 'stable'),
kube_pod(name: 'two', status: 'Running', track: 'canary')
])
rollout_instances = described_class.new(deployments, pods)
expect(rollout_instances.pod_instances).to eq([{
pod_name: 'two',
stable: false,
status: 'running',
tooltip: 'two (Running)',
track: 'canary'
}, {
pod_name: 'one',
stable: true,
status: 'running',
tooltip: 'one (Running)',
track: 'stable'
}])
end
end
end
......@@ -28,199 +28,241 @@ RSpec.describe Gitlab::Kubernetes::RolloutStatus do
subject(:rollout_status) { described_class.from_deployments(*specs, pods_attrs: pods) }
describe '#deployments' do
it 'stores the deployments' do
expect(rollout_status.deployments).to be_kind_of(Array)
expect(rollout_status.deployments.size).to eq(2)
expect(rollout_status.deployments.first).to be_kind_of(::Gitlab::Kubernetes::Deployment)
shared_examples 'rollout status' do
describe '#deployments' do
it 'stores the deployments' do
expect(rollout_status.deployments).to be_kind_of(Array)
expect(rollout_status.deployments.size).to eq(2)
expect(rollout_status.deployments.first).to be_kind_of(::Gitlab::Kubernetes::Deployment)
end
end
end
describe '#instances' do
context 'for stable track' do
let(:track) { "any" }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: "any")
describe '#instances' do
context 'for stable track' do
let(:track) { "any" }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: "any")
end
it 'stores the union of deployment instances' do
expected = [
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true }
]
expect(rollout_status.instances).to eq(expected)
end
end
it 'stores the union of deployment instances' do
expected = [
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'any', stable: false },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true }
]
expect(rollout_status.instances).to eq(expected)
context 'for stable track' do
let(:track) { 'canary' }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track)
end
it 'sorts stable instances last' do
expected = [
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true }
]
expect(rollout_status.instances).to eq(expected)
end
end
end
context 'for stable track' do
let(:track) { 'canary' }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track)
end
describe '#completion' do
subject { rollout_status.completion }
it 'sorts stable instances last' do
expected = [
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "two", tooltip: 'two (Running)', track: 'canary', stable: false },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true },
{ status: 'running', pod_name: "one", tooltip: 'one (Running)', track: 'stable', stable: true }
]
context 'when all instances are finished' do
let(:track) { 'canary' }
expect(rollout_status.instances).to eq(expected)
it { is_expected.to eq(100) }
end
end
end
describe '#completion' do
subject { rollout_status.completion }
context 'when half of the instances are finished' do
let(:track) { "canary" }
context 'when all instances are finished' do
let(:track) { 'canary' }
it { is_expected.to eq(100) }
end
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track, status: "Pending")
end
context 'when half of the instances are finished' do
let(:track) { "canary" }
let(:specs) { specs_half_finished }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track, status: "Pending")
it { is_expected.to eq(50) }
end
let(:specs) { specs_half_finished }
context 'with one deployment' do
it 'sets the completion percentage when a deployment has more running pods than desired' do
deployments = [kube_deployment(name: 'one', track: 'one', replicas: 2)]
pods = create_pods(name: 'one', track: 'one', count: 3)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
it { is_expected.to eq(50) }
end
context 'with one deployment' do
it 'sets the completion percentage when a deployment has more running pods than desired' do
deployments = [kube_deployment(name: 'one', track: 'one', replicas: 2)]
pods = create_pods(name: 'one', track: 'one', count: 3)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(100)
end
end
expect(rollout_status.completion).to eq(100)
context 'with two deployments on different tracks' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'one', replicas: 2),
kube_deployment(name: 'two', track: 'two', replicas: 2)
]
pods = create_pods(name: 'one', track: 'one', count: 2) + create_pods(name: 'two', track: 'two', count: 2)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(100)
end
end
end
context 'with two deployments on different tracks' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'one', replicas: 2),
kube_deployment(name: 'two', track: 'two', replicas: 2)
]
pods = create_pods(name: 'one', track: 'one', count: 2) + create_pods(name: 'two', track: 'two', count: 2)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
context 'with two deployments that both have track set to "stable"' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 2),
kube_deployment(name: 'two', track: 'stable', replicas: 2)
]
pods = create_pods(name: 'one', track: 'stable', count: 2) + create_pods(name: 'two', track: 'stable', count: 2)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(100)
end
it 'sets the completion percentage when no pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 3),
kube_deployment(name: 'two', track: 'stable', replicas: 7)
]
rollout_status = described_class.from_deployments(*deployments, pods_attrs: [])
expect(rollout_status.completion).to eq(0)
end
end
expect(rollout_status.completion).to eq(100)
context 'with two deployments, one with track set to "stable" and one with no track label' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 3),
kube_deployment(name: 'two', track: nil, replicas: 3)
]
pods = create_pods(name: 'one', track: 'stable', count: 3) + create_pods(name: 'two', track: nil, count: 3)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(100)
end
it 'sets the completion percentage when no pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 1),
kube_deployment(name: 'two', track: nil, replicas: 1)
]
rollout_status = described_class.from_deployments(*deployments, pods_attrs: [])
expect(rollout_status.completion).to eq(0)
end
end
end
context 'with two deployments that both have track set to "stable"' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 2),
kube_deployment(name: 'two', track: 'stable', replicas: 2)
]
pods = create_pods(name: 'one', track: 'stable', count: 2) + create_pods(name: 'two', track: 'stable', count: 2)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
describe '#complete?' do
subject { rollout_status.complete? }
expect(rollout_status.completion).to eq(100)
context 'when all instances are finished' do
let(:track) { 'canary' }
it { is_expected.to be_truthy }
end
it 'sets the completion percentage when no pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 3),
kube_deployment(name: 'two', track: 'stable', replicas: 7)
]
rollout_status = described_class.from_deployments(*deployments, pods_attrs: [])
context 'when half of the instances are finished' do
let(:track) { "canary" }
expect(rollout_status.completion).to eq(0)
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track, status: "Pending")
end
let(:specs) { specs_half_finished }
it { is_expected.to be_falsy}
end
end
context 'with two deployments, one with track set to "stable" and one with no track label' do
it 'sets the completion percentage when all pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 3),
kube_deployment(name: 'two', track: nil, replicas: 3)
]
pods = create_pods(name: 'one', track: 'stable', count: 3) + create_pods(name: 'two', track: nil, count: 3)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(100)
describe '#found?' do
context 'when the specs are passed' do
it { is_expected.to be_found }
end
it 'sets the completion percentage when no pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 1),
kube_deployment(name: 'two', track: nil, replicas: 1)
]
rollout_status = described_class.from_deployments(*deployments, pods_attrs: [])
context 'when list of specs is empty' do
let(:specs) { [] }
expect(rollout_status.completion).to eq(0)
it { is_expected.not_to be_found }
end
end
end
describe '#complete?' do
subject { rollout_status.complete? }
describe '.loading' do
subject { described_class.loading }
context 'when all instances are finished' do
let(:track) { 'canary' }
it { is_expected.to be_truthy }
it { is_expected.to be_loading }
end
context 'when half of the instances are finished' do
let(:track) { "canary" }
let(:pods) do
create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: track, status: "Pending")
describe '#not_found?' do
context 'when the specs are passed' do
it { is_expected.not_to be_not_found }
end
let(:specs) { specs_half_finished }
context 'when list of specs is empty' do
let(:specs) { [] }
it { is_expected.to be_falsy}
it { is_expected.to be_not_found }
end
end
end
describe '#not_found?' do
context 'when the specs are passed' do
it { is_expected.not_to be_not_found }
context 'deploy_boards_dedupe_instances is disabled' do
before do
stub_feature_flags(deploy_boards_dedupe_instances: false)
end
context 'when list of specs is empty' do
let(:specs) { [] }
it { is_expected.to be_not_found }
end
it_behaves_like 'rollout status'
end
describe '#found?' do
context 'when the specs are passed' do
it { is_expected.to be_found }
context 'deploy_boards_dedupe_instances is enabled' do
before do
stub_feature_flags(deploy_boards_dedupe_instances: true)
end
context 'when list of specs is empty' do
let(:specs) { [] }
it_behaves_like 'rollout status'
it { is_expected.not_to be_found }
end
end
describe '#completion' do
it 'sets the completion percentage when a quarter of the pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 6),
kube_deployment(name: 'two', track: 'stable', replicas: 2)
]
pods = create_pods(name: 'one', track: 'stable', count: 2)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
expect(rollout_status.completion).to eq(25)
end
describe '.loading' do
subject { described_class.loading }
it 'sets the completion percentage when a third of the pods are complete' do
deployments = [
kube_deployment(name: 'one', track: 'stable', replicas: 2),
kube_deployment(name: 'two', track: nil, replicas: 7)
]
pods = create_pods(name: 'one', track: 'stable', count: 2) + create_pods(name: 'two', track: nil, count: 1)
rollout_status = described_class.from_deployments(*deployments, pods_attrs: pods)
it { is_expected.to be_loading }
expect(rollout_status.completion).to eq(33)
end
end
end
def create_pods(name:, count:, track: nil, status: 'Running' )
......
......@@ -276,6 +276,28 @@ RSpec.describe Clusters::Platforms::Kubernetes do
])
end
end
context 'with multiple matching deployments' do
let(:deployments) do
[
kube_deployment(name: 'deployment-a', environment_slug: environment.slug, project_slug: project.full_path_slug, replicas: 2),
kube_deployment(name: 'deployment-b', environment_slug: environment.slug, project_slug: project.full_path_slug, replicas: 2)
]
end
let(:pods) do
[
kube_pod(name: 'pod-a-1', environment_slug: environment.slug, project_slug: project.full_path_slug),
kube_pod(name: 'pod-a-2', environment_slug: environment.slug, project_slug: project.full_path_slug),
kube_pod(name: 'pod-b-1', environment_slug: environment.slug, project_slug: project.full_path_slug),
kube_pod(name: 'pod-b-2', environment_slug: environment.slug, project_slug: project.full_path_slug)
]
end
it 'returns each pod once' do
expect(rollout_status.instances.map { |p| p[:pod_name] }).to eq(['pod-a-1', 'pod-a-2', 'pod-b-1', 'pod-b-2'])
end
end
end
describe '#calculate_reactive_cache_for' do
......
......@@ -2,13 +2,47 @@
module Gitlab
module Kubernetes
module Pod
class Pod
PENDING = 'Pending'
RUNNING = 'Running'
SUCCEEDED = 'Succeeded'
FAILED = 'Failed'
UNKNOWN = 'Unknown'
PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze
STABLE_TRACK_VALUE = 'stable'
def initialize(attributes = {})
@attributes = attributes
end
def track
attributes.dig('metadata', 'labels', 'track') || STABLE_TRACK_VALUE
end
def name
metadata['name'] || metadata['generateName']
end
def stable?
track == STABLE_TRACK_VALUE
end
def status
attributes.dig('status', 'phase')
end
def order
stable? ? 1 : 0
end
private
attr_reader :attributes
def metadata
attributes.fetch('metadata', {})
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment