Merge branch 'unquarantine-agent-kubernetes-test' into 'master'

Unquarantine kubernetes tests

See merge request gitlab-org/gitlab!73187

app/views/clusters/clusters/_cluster_list.html.haml

@@ -2,7 +2,7 @@
   .top-area.adjust
     .gl-display-block.gl-text-right.gl-my-4.gl-w-full
       - if clusterable.can_add_cluster?
-        = link_to s_('ClusterIntegration|Connect cluster with certificate'), clusterable.new_path, class: 'btn gl-button btn-confirm js-add-cluster gl-py-2', qa_selector: :integrate_kubernetes_cluster_button
+        = link_to s_('ClusterIntegration|Connect cluster with certificate'), clusterable.new_path, class: 'btn gl-button btn-confirm js-add-cluster gl-py-2', data: { qa_selector: 'integrate_kubernetes_cluster_button' }
       - else
         %span.btn.gl-button.btn-confirm.js-add-cluster.disabled.gl-py-2
           = s_("ClusterIntegration|Connect cluster with certificate")

qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb

@@ -24,8 +24,7 @@ spec:
           args:
             - --token-file=/config/token
             - --kas-address
-            - "<%= kas_wss_address %>" # Use this for GitLab chart deployments
-            # - "<%= kas_grpc_address %>" # Use this for GDK
+            - "<%= kas_wss_address %>"
           volumeMounts:
             - name: token-volume
               mountPath: /config

qa/qa/page/project/infrastructure/kubernetes/add.rb

@@ -11,7 +11,7 @@ module QA
             end
 
             def add_existing_cluster
-              click_element(:add_existing_cluster_tab)
+              page.find('.gl-tab-nav-item', text: 'Connect existing cluster').click
             end
           end
         end

qa/qa/page/project/infrastructure/kubernetes/index.rb

@@ -6,12 +6,12 @@ module QA
       module Infrastructure
         module Kubernetes
           class Index < Page::Base
-            view 'app/assets/javascripts/clusters_list/components/clusters_empty_state.vue' do
-              element :add_kubernetes_cluster_link
+            view 'app/views/clusters/clusters/_cluster_list.html.haml' do
+              element :integrate_kubernetes_cluster_button
             end
 
-            def add_kubernetes_cluster
-              click_element :add_kubernetes_cluster_link
+            def connect_cluster_with_certificate
+              find('.js-add-cluster').click
             end
 
             def has_cluster?(cluster)

qa/qa/page/project/infrastructure/kubernetes/show.rb

@@ -7,26 +7,9 @@ module QA
         module Kubernetes
           class Show < Page::Base
             view 'app/assets/javascripts/clusters/forms/components/integration_form.vue' do
-              element :integration_status_toggle, required: true
-              element :base_domain_field, required: true
-              element :save_changes_button, required: true
-            end
-
-            view 'app/views/clusters/clusters/_details_tab.html.haml' do
-              element :details, required: true
-            end
-
-            view 'app/views/clusters/clusters/_health.html.haml' do
-              element :cluster_health_section
-            end
-
-            view 'app/views/clusters/clusters/_health_tab.html.haml' do
-              element :health, required: true
-            end
-
-            def open_details
-              has_element?(:details, wait: 30)
-              click_element :details
+              element :integration_status_toggle
+              element :base_domain_field
+              element :save_changes_button
             end
 
             def set_domain(domain)
@@ -36,29 +19,6 @@ module QA
             def save_domain
               click_element :save_changes_button, Page::Project::Infrastructure::Kubernetes::Show
             end
-
-            def wait_for_cluster_health
-              wait_until(max_duration: 120, sleep_interval: 3, reload: true) do
-                has_cluster_health_graphs?
-              end
-            end
-
-            def open_health
-              has_element?(:health, wait: 30)
-              click_element :health
-            end
-
-            def has_cluster_health_graphs?
-              within_cluster_health_section do
-                has_text?('CPU Usage')
-              end
-            end
-
-            def within_cluster_health_section
-              within_element :cluster_health_section do
-                yield
-              end
-            end
           end
         end
       end

qa/qa/resource/clusters/agent.rb

@@ -19,13 +19,12 @@ module QA
         def fabricate!
           puts 'TODO: FABRICATE VIA UI'
         end
-        # TODO
-        #
-        # The UI for this model is not yet implemented. So far it can only be
-        # created through the GraphQL API
-        # def fabricate
-        #
-        # end
+
+        def resource_web_url(resource)
+          super
+        rescue ResourceURLMissingError
+          # this particular resource does not expose a web_url property
+        end
 
         def api_get_path
           "gid://gitlab/Clusters::Agent/#{id}"

qa/qa/resource/clusters/agent_token.rb

@@ -13,13 +13,12 @@ module QA
         def fabricate!
           puts 'TODO: FABRICATE VIA UI'
         end
-        # TODO
-        #
-        # The UI for this model is not yet implemented. So far it can only be
-        # created through the GraphQL API
-        # def fabricate
-        #
-        # end
+
+        def resource_web_url(resource)
+          super
+        rescue ResourceURLMissingError
+          # this particular resource does not expose a web_url property
+        end
 
         def api_get_path
           "gid://gitlab/Clusters::AgentToken/#{id}"

qa/qa/resource/kubernetes_cluster/project_cluster.rb

@@ -13,8 +13,8 @@ module QA
           Resource::Project.fabricate!
         end
 
-        def ingress_ip
-          @ingress_ip ||= @cluster.fetch_external_ip_for_ingress
+        attribute :ingress_ip do
+          @cluster.fetch_external_ip_for_ingress
         end
 
         def fabricate!
@@ -24,7 +24,7 @@ module QA
             &:go_to_infrastructure_kubernetes)
 
           Page::Project::Infrastructure::Kubernetes::Index.perform(
-            &:add_kubernetes_cluster)
+            &:connect_cluster_with_certificate)
 
           Page::Project::Infrastructure::Kubernetes::Add.perform(
             &:add_existing_cluster)
@@ -39,14 +39,10 @@ module QA
           end
 
           Page::Project::Infrastructure::Kubernetes::Show.perform do |show|
-            # We must wait a few seconds for permissions to be set up correctly for new cluster
-            sleep 25
-
             if @install_ingress
-              populate(:ingress_ip)
+              ingress_ip
 
-              show.open_details
-              show.set_domain("#{ingress_ip}.nip.io")
+              show.set_domain("#{@ingress_ip}.nip.io")
               show.save_domain
             end
           end

qa/qa/runtime/env.rb

@@ -403,7 +403,7 @@ module QA
       end
 
       def gitlab_agentk_version
-        ENV.fetch('GITLAB_AGENTK_VERSION', 'v13.7.0')
+        ENV.fetch('GITLAB_AGENTK_VERSION', 'v14.4.0')
       end
 
       def transient_trials

qa/qa/service/cluster_provider/gcloud.rb

@@ -24,16 +24,6 @@ module QA
           )
         end
 
-        def set_credentials(admin_user)
-          master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{@region} --format 'json(masterAuth.username, masterAuth.password)'`)
-
-          shell <<~CMD.tr("\n", ' ')
-            kubectl config set-credentials #{admin_user}
-              --username #{master_auth['masterAuth']['username']}
-              --password #{master_auth['masterAuth']['password']}
-          CMD
-        end
-
         def setup
           login_if_not_already_logged_in
           create_cluster
@@ -43,6 +33,12 @@ module QA
           delete_cluster
         end
 
+        def install_ingress
+          QA::Runtime::Logger.info "Attempting to install Ingress on cluster #{cluster_name}"
+          shell 'kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-0.31.0/deploy/static/provider/cloud/deploy.yaml'
+          wait_for_ingress
+        end
+
         private
 
         def login_if_not_already_logged_in
@@ -59,7 +55,7 @@ module QA
         end
 
         def attempt_login_with_env_vars
-          puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
+          QA::Runtime::Logger.debug("Logging in with GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY.")
           gcloud_account_key = Tempfile.new('gcloud-account-key')
           gcloud_account_key.write(Runtime::Env.gcloud_account_key)
           gcloud_account_key.close
@@ -80,7 +76,6 @@ module QA
             gcloud container clusters
             create #{cluster_name}
             #{auth_options}
-            --enable-basic-auth
             --region #{@region}
             --disk-size 10GB
             --num-nodes #{Runtime::Env.gcloud_num_nodes}
@@ -109,6 +104,18 @@ module QA
         def get_region
           Runtime::Env.gcloud_region || @available_regions.delete(@available_regions.sample)
         end
+
+        def wait_for_ingress
+          QA::Runtime::Logger.info 'Waiting for Ingress controller pod to be initialized'
+
+          Support::Retrier.retry_until(max_attempts: 60, sleep_interval: 1) do
+            service_available?('kubectl get pods --all-namespaces -l app.kubernetes.io/component=controller | grep -o "ingress-nginx-controller.*1/1"')
+          end
+        end
+
+        def service_available?(command)
+          system("#{command} > /dev/null 2>&1")
+        end
       end
     end
   end

qa/qa/service/kubernetes_cluster.rb

@@ -41,6 +41,10 @@ module QA
         cluster_name
       end
 
+      def install_ingress
+        @provider.install_ingress
+      end
+
       def create_secret(secret, secret_name)
         shell("kubectl create secret generic #{secret_name} --from-literal=token='#{secret}'")
       end
@@ -70,7 +74,13 @@ module QA
       end
 
       def fetch_external_ip_for_ingress
-        `kubectl get svc --all-namespaces --no-headers=true -l  app.kubernetes.io/name=ingress-nginx -o custom-columns=:'status.loadBalancer.ingress[0].ip' | grep -v 'none'`
+        install_ingress
+
+        # need to wait since the ingress-nginx service has an initial delay set of 10 seconds
+        sleep 10
+        ingress_ip = `kubectl get svc --all-namespaces --no-headers=true -l  app.kubernetes.io/name=ingress-nginx -o custom-columns=:'status.loadBalancer.ingress[0].ip' | grep -v 'none'`
+        QA::Runtime::Logger.debug "Has ingress address set to: #{ingress_ip}"
+        ingress_ip
       end
 
       private
@@ -82,7 +92,6 @@ module QA
       def fetch_credentials
         return global_credentials unless rbac
 
-        @provider.set_credentials(admin_user)
         create_service_account(admin_user)
         account_credentials
       end

qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb

 # frozen_string_literal: true
 
 module QA
-  RSpec.describe 'Configure' do
+  RSpec.describe 'Configure', only: { subdomain: :staging } do
     let(:project) do
       Resource::Project.fabricate_via_api! do |project|
-        project.name = Runtime::Env.auto_devops_project_name || 'autodevops-project'
+        project.name = 'autodevops-project'
         project.auto_devops_enabled = true
       end
     end
@@ -13,35 +13,24 @@ module QA
       disable_optional_jobs(project)
     end
 
-    describe 'Auto DevOps support', :orchestrated, :kubernetes, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/251090', type: :stale } do
+    describe 'Auto DevOps support' do
       context 'when rbac is enabled' do
         let(:cluster) { Service::KubernetesCluster.new.create! }
 
         after do
           cluster&.remove!
+          project.remove_via_api!
         end
 
         it 'runs auto devops', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1422' do
           Flow::Login.sign_in
 
-          # Set an application secret CI variable (prefixed with K8S_SECRET_)
-          Resource::CiVariable.fabricate! do |resource|
-            resource.project = project
-            resource.key = 'K8S_SECRET_OPTIONAL_MESSAGE'
-            resource.value = 'you_can_see_this_variable'
-            resource.masked = false
-          end
-
-          # Connect K8s cluster
           Resource::KubernetesCluster::ProjectCluster.fabricate! do |k8s_cluster|
             k8s_cluster.project = project
             k8s_cluster.cluster = cluster
             k8s_cluster.install_ingress = true
-            k8s_cluster.install_prometheus = true
-            k8s_cluster.install_runner = true
           end
 
-          # Create Auto DevOps compatible repo
           Resource::Repository::ProjectPush.fabricate! do |push|
             push.project = project
             push.directory = Pathname
@@ -78,46 +67,6 @@ module QA
 
             job.click_element(:pipeline_path)
           end
-
-          Page::Project::Menu.perform(&:go_to_deployments_environments)
-          Page::Project::Deployments::Environments::Index.perform do |index|
-            index.click_environment_link('production')
-          end
-          Page::Project::Deployments::Environments::Show.perform do |show|
-            show.view_deployment do
-              expect(page).to have_content('Hello World!')
-              expect(page).to have_content('you_can_see_this_variable')
-            end
-          end
-        end
-      end
-    end
-
-    describe 'Auto DevOps', :smoke do
-      before do
-        Flow::Login.sign_in
-
-        project.visit!
-
-        Page::Project::Menu.perform(&:go_to_ci_cd_settings)
-        Page::Project::Settings::CiCd.perform(&:expand_auto_devops)
-        Page::Project::Settings::AutoDevops.perform(&:enable_autodevops)
-
-        # Create AutoDevOps repo
-        Resource::Repository::ProjectPush.fabricate! do |push|
-          push.project = project
-          push.directory = Pathname
-            .new(__dir__)
-            .join('../../../../../fixtures/auto_devops_rack')
-          push.commit_message = 'Create AutoDevOps compatible Project'
-        end
-      end
-
-      it 'runs an AutoDevOps pipeline', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1564' do
-        Flow::Pipeline.visit_latest_pipeline
-
-        Page::Project::Pipeline::Show.perform do |pipeline|
-          expect(pipeline).to have_tag('Auto DevOps')
         end
       end
     end
@@ -128,7 +77,8 @@ module QA
       %w[
         CODE_QUALITY_DISABLED LICENSE_MANAGEMENT_DISABLED
         SAST_DISABLED DAST_DISABLED DEPENDENCY_SCANNING_DISABLED
-        CONTAINER_SCANNING_DISABLED
+        CONTAINER_SCANNING_DISABLED BROWSER_PERFORMANCE_DISABLED
+        SECRET_DETECTION_DISABLED
       ].each do |key|
         Resource::CiVariable.fabricate_via_api! do |resource|
           resource.project = project

qa/qa/specs/features/ee/api/7_configure/kubernetes/kubernetes_agent_spec.rb

@@ -2,11 +2,11 @@
 require 'erb'
 
 module QA
-  RSpec.describe 'Configure' do
+  RSpec.describe 'Configure', only: { subdomain: :staging } do
     include Service::Shellout
 
-    describe 'Kubernetes Agent', :orchestrated, :kubernetes, :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/294177', type: :waiting_on } do
-      let!(:cluster) { Service::KubernetesCluster.new(provider_class: Service::ClusterProvider::K3s).create! }
+    describe 'Kubernetes Agent' do
+      let!(:cluster) { Service::KubernetesCluster.new(provider_class: Service::ClusterProvider::Gcloud).create! }
 
       let(:agent_token) do
         Resource::Clusters::AgentToken.fabricate_via_api!
@@ -23,7 +23,7 @@ module QA
       end
 
       after do
-        cluster.remove!
+        cluster&.remove!
 
         project.group.remove_via_api!
       end
@@ -43,10 +43,7 @@ module QA
       def install_agentk(cluster, agent_token)
         cluster.create_secret(agent_token.secret, 'gitlab-agent-token')
 
-        uri = URI.parse(Runtime::Scenario.gitlab_address)
-
-        kas_grpc_address = "grpc://#{uri.host}:8150"
-        kas_wss_address = "wss://kas.#{uri.host}:#{uri.port}"
+        kas_wss_address = "wss://kas.staging.gitlab.com"
         agent_manifest_template = read_agent_fixture('agentk-manifest.yaml.erb')
         agent_manifest_yaml = ERB.new(agent_manifest_template).result(binding)