Skip to content

G
gitlab-ce
Commit b4c2212c authored by Dmytro Zaporozhets (DZ)'s avatar Dmytro Zaporozhets (DZ)
Browse files
Options

Merge branch 'remove-cluster-applications-artifact' into 'master' 

Remove cluster applications report and parser

See merge request gitlab-org/gitlab!63897
parents 5f0dd3b1 b6181d8e
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 529 deletions
app/models/ci/job_artifact.rb
View file @ b4c2212c
...@@ -43,7 +43,7 @@ module Ci ...@@ -43,7 +43,7 @@ module Ci
dotenv: '.env', dotenv: '.env',
cobertura: 'cobertura-coverage.xml', cobertura: 'cobertura-coverage.xml',
terraform: 'tfplan.json', terraform: 'tfplan.json',
cluster_applications: 'gl-cluster-applications.json', cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441
requirements: 'requirements.json', requirements: 'requirements.json',
coverage_fuzzing: 'gl-coverage-fuzzing.json', coverage_fuzzing: 'gl-coverage-fuzzing.json',
api_fuzzing: 'gl-api-fuzzing-report.json' api_fuzzing: 'gl-api-fuzzing-report.json'
......
app/services/ci/job_artifacts/create_service.rb
View file @ b4c2212c
...@@ -115,7 +115,6 @@ module Ci ...@@ -115,7 +115,6 @@ module Ci
case artifact.file_type case artifact.file_type
when 'dotenv' then parse_dotenv_artifact(artifact) when 'dotenv' then parse_dotenv_artifact(artifact)
when 'cluster_applications' then parse_cluster_applications_artifact(artifact)
else success else success
end end
end end
...@@ -165,10 +164,6 @@ module Ci ...@@ -165,10 +164,6 @@ module Ci
def parse_dotenv_artifact(artifact) def parse_dotenv_artifact(artifact)
Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact) Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact)
end end
def parse_cluster_applications_artifact(artifact)
Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact)
end
end end
end end
end end
app/services/clusters/parse_cluster_applications_artifact_service.rb deleted 100644 → 0
View file @ 5f0dd3b1
# frozen_string_literal: true
module Clusters
class ParseClusterApplicationsArtifactService < ::BaseService
include Gitlab::Utils::StrongMemoize
MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes
RELEASE_NAMES = %w[cilium].freeze
def initialize(job, current_user)
@job = job
super(job.project, current_user)
end
def execute(artifact)
raise ArgumentError, 'Artifact is not cluster_applications file type' unless artifact&.cluster_applications?
return error(too_big_error_message, :bad_request) unless artifact.file.size < MAX_ACCEPTABLE_ARTIFACT_SIZE
return error(no_deployment_message, :bad_request) unless job.deployment
return error(no_deployment_cluster_message, :bad_request) unless cluster
parse!(artifact)
success
rescue Gitlab::Kubernetes::Helm::Parsers::ListV2::ParserError, ActiveRecord::RecordInvalid => error
Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
error(error.message, :bad_request)
end
private
attr_reader :job
def cluster
strong_memoize(:cluster) do
deployment_cluster = job.deployment&.cluster
deployment_cluster if Ability.allowed?(current_user, :admin_cluster, deployment_cluster)
end
end
def parse!(artifact)
releases = []
artifact.each_blob do |blob|
next if blob.empty?
releases.concat(Gitlab::Kubernetes::Helm::Parsers::ListV2.new(blob).releases)
end
update_cluster_application_statuses!(releases)
end
def update_cluster_application_statuses!(releases)
release_by_name = releases.index_by { |release| release['Name'] }
Clusters::Cluster.transaction do
RELEASE_NAMES.each do |release_name|
application_class = Clusters::Cluster::APPLICATIONS[release_name]
application = cluster.find_or_build_application(application_class)
release = release_by_name[release_name]
if release
case release['Status']
when 'DEPLOYED'
application.make_externally_installed!
when 'FAILED'
application.make_errored!(s_('ClusterIntegration|Helm release failed to install'))
end
else
# missing, so by definition, we consider this uninstalled
application.make_externally_uninstalled! if application.persisted?
end
end
end
end
def too_big_error_message
human_size = ActiveSupport::NumberHelper.number_to_human_size(MAX_ACCEPTABLE_ARTIFACT_SIZE)
s_('ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}') % { human_size: human_size }
end
def no_deployment_message
s_('ClusterIntegration|No deployment found for this job')
end
def no_deployment_cluster_message
s_('ClusterIntegration|No deployment cluster found for this job')
end
end
end
lib/gitlab/ci/config/entry/reports.rb
View file @ b4c2212c
...@@ -43,7 +43,7 @@ module Gitlab ...@@ -43,7 +43,7 @@ module Gitlab
validates :cobertura, array_of_strings_or_string: true validates :cobertura, array_of_strings_or_string: true
validates :terraform, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true
validates :accessibility, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true
validates :cluster_applications, array_of_strings_or_string: true validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441
validates :requirements, array_of_strings_or_string: true validates :requirements, array_of_strings_or_string: true
end end
end end
......
lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
View file @ b4c2212c
...@@ -27,8 +27,6 @@ apply: ...@@ -27,8 +27,6 @@ apply:
variables: variables:
- $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
artifacts: artifacts:
reports:
cluster_applications: gl-cluster-applications.json
when: on_failure when: on_failure
paths: paths:
- tiller.log - tiller.log
lib/gitlab/kubernetes/helm/parsers/list_v2.rb deleted 100644 → 0
View file @ 5f0dd3b1
# frozen_string_literal: true
module Gitlab
module Kubernetes
module Helm
module Parsers
# Parses Helm v2 list (JSON) output
class ListV2
ParserError = Class.new(StandardError)
attr_reader :contents, :json
def initialize(contents)
@contents = contents
@json = Gitlab::Json.parse(contents)
rescue JSON::ParserError => e
raise ParserError, e.message
end
def releases
@releases = helm_releases
end
private
def helm_releases
helm_releases = json['Releases'] || []
raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) }
helm_releases
end
end
end
end
end
end
locale/gitlab.pot
View file @ b4c2212c
...@@ -7072,9 +7072,6 @@ msgstr "" ...@@ -7072,9 +7072,6 @@ msgstr ""
msgid "ClusterIntegration|Cluster name is required." msgid "ClusterIntegration|Cluster name is required."
msgstr "" msgstr ""
msgid "ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}"
msgstr ""
msgid "ClusterIntegration|Clusters are utilized by selecting the nearest ancestor with a matching environment scope. For example, project clusters will override group clusters. %{linkStart}More information%{linkEnd}" msgid "ClusterIntegration|Clusters are utilized by selecting the nearest ancestor with a matching environment scope. For example, project clusters will override group clusters. %{linkStart}More information%{linkEnd}"
msgstr "" msgstr ""
...@@ -7279,9 +7276,6 @@ msgstr "" ...@@ -7279,9 +7276,6 @@ msgstr ""
msgid "ClusterIntegration|HTTP Error" msgid "ClusterIntegration|HTTP Error"
msgstr "" msgstr ""
msgid "ClusterIntegration|Helm release failed to install"
msgstr ""
msgid "ClusterIntegration|If you are setting up multiple clusters and are using Auto DevOps, %{help_link_start}read this first%{help_link_end}." msgid "ClusterIntegration|If you are setting up multiple clusters and are using Auto DevOps, %{help_link_start}read this first%{help_link_end}."
msgstr "" msgstr ""
...@@ -7441,12 +7435,6 @@ msgstr "" ...@@ -7441,12 +7435,6 @@ msgstr ""
msgid "ClusterIntegration|No VPCs found" msgid "ClusterIntegration|No VPCs found"
msgstr "" msgstr ""
msgid "ClusterIntegration|No deployment cluster found for this job"
msgstr ""
msgid "ClusterIntegration|No deployment found for this job"
msgstr ""
msgid "ClusterIntegration|No instance type found" msgid "ClusterIntegration|No instance type found"
msgstr "" msgstr ""
......
spec/factories/ci/job_artifacts.rb
View file @ b4c2212c
...@@ -317,21 +317,6 @@ FactoryBot.define do ...@@ -317,21 +317,6 @@ FactoryBot.define do
end end
end end
trait :cluster_applications do
file_type { :cluster_applications }
file_format { :gzip }
transient do
file do
fixture_file_upload(Rails.root.join('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz'), 'application/x-gzip')
end
end
after(:build) do |artifact, evaluator|
artifact.file = evaluator.file
end
end
trait :correct_checksum do trait :correct_checksum do
after(:build) do |artifact, evaluator| after(:build) do |artifact, evaluator|
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
......
spec/lib/gitlab/kubernetes/helm/parsers/list_v2_spec.rb deleted 100644 → 0
View file @ 5f0dd3b1
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Kubernetes::Helm::Parsers::ListV2 do
let(:valid_file_contents) do
<<~EOF
{
"Next": "",
"Releases": [
{
"Name": "certmanager",
"Revision": 2,
"Updated": "Sun Mar 29 06:55:42 2020",
"Status": "DEPLOYED",
"Chart": "cert-manager-v0.10.1",
"AppVersion": "v0.10.1",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "certmanager-crds",
"Revision": 2,
"Updated": "Sun Mar 29 06:55:32 2020",
"Status": "DEPLOYED",
"Chart": "cert-manager-crds-v0.2.0",
"AppVersion": "release-0.10",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "certmanager-issuer",
"Revision": 1,
"Updated": "Tue Feb 18 10:04:04 2020",
"Status": "FAILED",
"Chart": "cert-manager-issuer-v0.1.0",
"AppVersion": "",
"Namespace": "gitlab-managed-apps"
},
{
"Name": "runner",
"Revision": 2,
"Updated": "Sun Mar 29 07:01:01 2020",
"Status": "DEPLOYED",
"Chart": "gitlab-runner-0.14.0",
"AppVersion": "12.8.0",
"Namespace": "gitlab-managed-apps"
}
]
}
EOF
end
describe '#initialize' do
it 'initializes without error' do
expect do
described_class.new(valid_file_contents)
end.not_to raise_error
end
it 'raises an error on invalid JSON' do
expect do
described_class.new('')
end.to raise_error(described_class::ParserError)
end
end
describe '#releases' do
subject(:list_v2) { described_class.new(valid_file_contents) }
it 'returns list of releases' do
expect(list_v2.releases).to match([
a_hash_including('Name' => 'certmanager', 'Status' => 'DEPLOYED'),
a_hash_including('Name' => 'certmanager-crds', 'Status' => 'DEPLOYED'),
a_hash_including('Name' => 'certmanager-issuer', 'Status' => 'FAILED'),
a_hash_including('Name' => 'runner', 'Status' => 'DEPLOYED')
])
end
context 'empty Releases' do
let(:valid_file_contents) { '{}' }
it 'returns an empty array' do
expect(list_v2.releases).to eq([])
end
end
context 'invalid Releases' do
let(:invalid_file_contents) do
'{ "Releases" : ["a", "b"] }'
end
subject(:list_v2) { described_class.new(invalid_file_contents) }
it 'raises an error' do
expect do
list_v2.releases
end.to raise_error(described_class::ParserError, 'Invalid format for Releases')
end
end
end
end
spec/services/ci/job_artifacts/create_service_spec.rb
View file @ b4c2212c
...@@ -203,53 +203,6 @@ RSpec.describe Ci::JobArtifacts::CreateService do ...@@ -203,53 +203,6 @@ RSpec.describe Ci::JobArtifacts::CreateService do
end end
end end
context 'when artifact type is cluster_applications' do
let(:artifacts_file) do
file_to_upload('spec/fixtures/helm/helm_list_v2_prometheus_missing.json.gz', sha256: artifacts_sha256)
end
let(:params) do
{
'artifact_type' => 'cluster_applications',
'artifact_format' => 'gzip'
}.with_indifferent_access
end
it 'calls cluster applications parse service' do
expect_next_instance_of(Clusters::ParseClusterApplicationsArtifactService) do |service|
expect(service).to receive(:execute).once.and_call_original
end
subject
end
context 'when there is a deployment cluster' do
let(:user) { project.owner }
before do
job.update!(user: user)
end
it 'calls cluster applications parse service with job and job user', :aggregate_failures do
expect(Clusters::ParseClusterApplicationsArtifactService).to receive(:new).with(job, user).and_call_original
subject
end
end
context 'when ci_synchronous_artifact_parsing feature flag is disabled' do
before do
stub_feature_flags(ci_synchronous_artifact_parsing: false)
end
it 'does not call parse service' do
expect(Clusters::ParseClusterApplicationsArtifactService).not_to receive(:new)
expect(subject[:status]).to eq(:success)
end
end
end
shared_examples 'rescues object storage error' do |klass, message, expected_message| shared_examples 'rescues object storage error' do |klass, message, expected_message|
it "handles #{klass}" do it "handles #{klass}" do
allow_next_instance_of(JobArtifactUploader) do |uploader| allow_next_instance_of(JobArtifactUploader) do |uploader|
......
spec/services/clusters/parse_cluster_applications_artifact_service_spec.rb deleted 100644 → 0
View file @ 5f0dd3b1
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Clusters::ParseClusterApplicationsArtifactService do
let_it_be(:project) { create(:project) }
let_it_be(:user) { create(:user) }
before do
project.add_maintainer(user)
end
describe 'RELEASE_NAMES' do
it 'is included in Cluster application names', :aggregate_failures do
described_class::RELEASE_NAMES.each do |release_name|
expect(Clusters::Cluster::APPLICATIONS).to include(release_name)
end
end
end
describe '.new' do
let(:job) { build(:ci_build) }
it 'sets the project and current user', :aggregate_failures do
service = described_class.new(job, user)
expect(service.project).to eq(job.project)
expect(service.current_user).to eq(user)
end
end
describe '#execute' do
let_it_be(:cluster, reload: true) { create(:cluster, projects: [project]) }
let_it_be(:deployment, reload: true) { create(:deployment, cluster: cluster) }
let(:job) { deployment.deployable }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job) }
it 'calls Gitlab::Kubernetes::Helm::Parsers::ListV2' do
expect(Gitlab::Kubernetes::Helm::Parsers::ListV2).to receive(:new).and_call_original
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:success)
end
context 'artifact is not of cluster_applications type' do
let(:artifact) { create(:ci_job_artifact, :archive) }
let(:job) { artifact.job }
it 'raise ArgumentError' do
expect do
described_class.new(job, user).execute(artifact)
end.to raise_error(ArgumentError, 'Artifact is not cluster_applications file type')
end
end
context 'artifact exceeds acceptable size' do
it 'returns an error' do
stub_const("#{described_class}::MAX_ACCEPTABLE_ARTIFACT_SIZE", 1.byte)
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('Cluster_applications artifact too big. Maximum allowable size: 1 Byte')
end
end
context 'job has no deployment' do
let(:job) { build(:ci_build) }
it 'returns an error' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment found for this job')
end
end
context 'job has no deployment cluster' do
let(:deployment) { create(:deployment) }
let(:job) { deployment.deployable }
it 'returns an error' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment cluster found for this job')
end
end
context 'blob is empty' do
let(:file) { fixture_file_upload(Rails.root.join("spec/fixtures/helm/helm_list_v2_empty_blob.json.gz")) }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
it 'returns success' do
result = described_class.new(job, user).execute(artifact)
expect(result[:status]).to eq(:success)
end
end
context 'job has deployment cluster' do
context 'current user does not have access to deployment cluster' do
let(:other_user) { create(:user) }
it 'returns an error' do
result = described_class.new(job, other_user).execute(artifact)
expect(result[:status]).to eq(:error)
expect(result[:message]).to eq('No deployment cluster found for this job')
end
end
it 'does not affect unpermitted cluster applications' do
expect(Clusters::ParseClusterApplicationsArtifactService::RELEASE_NAMES).to contain_exactly('cilium')
end
Clusters::ParseClusterApplicationsArtifactService::RELEASE_NAMES.each do |release_name|
context release_name do
include_examples 'parse cluster applications artifact', release_name
end
end
end
end
end
spec/support/shared_examples/services/clusters/parse_cluster_applications_artifact_shared_examples.rb deleted 100644 → 0
View file @ 5f0dd3b1
# frozen_string_literal: true
RSpec.shared_examples 'parse cluster applications artifact' do |release_name|
let(:application_class) { Clusters::Cluster::APPLICATIONS[release_name] }
let(:cluster_application) { cluster.public_send("application_#{release_name}") }
let(:file) { fixture_file_upload(Rails.root.join(fixture)) }
let(:artifact) { create(:ci_job_artifact, :cluster_applications, job: job, file: file) }
context 'release is missing' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_missing.json.gz" }
context 'application does not exist' do
it 'does not create or destroy an application' do
expect do
described_class.new(job, user).execute(artifact)
end.not_to change(application_class, :count)
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :installed, cluster: cluster)
end
it 'marks the application as uninstalled' do
described_class.new(job, user).execute(artifact)
cluster_application.reload
expect(cluster_application).to be_uninstalled
end
end
end
context 'release is deployed' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_deployed.json.gz" }
context 'application does not exist' do
it 'creates an application and marks it as installed' do
expect do
described_class.new(job, user).execute(artifact)
end.to change(application_class, :count)
expect(cluster_application).to be_persisted
expect(cluster_application).to be_externally_installed
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :errored, cluster: cluster)
end
it 'marks the application as installed' do
described_class.new(job, user).execute(artifact)
expect(cluster_application).to be_externally_installed
end
end
end
context 'release is failed' do
let(:fixture) { "spec/fixtures/helm/helm_list_v2_#{release_name}_failed.json.gz" }
context 'application does not exist' do
it 'creates an application and marks it as errored' do
expect do
described_class.new(job, user).execute(artifact)
end.to change(application_class, :count)
expect(cluster_application).to be_persisted
expect(cluster_application).to be_errored
expect(cluster_application.status_reason).to eq('Helm release failed to install')
end
end
context 'application exists' do
before do
create("clusters_applications_#{release_name}".to_sym, :installed, cluster: cluster)
end
it 'marks the application as errored' do
described_class.new(job, user).execute(artifact)
expect(cluster_application).to be_errored
expect(cluster_application.status_reason).to eq('Helm release failed to install')
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7