Creating a gitaly ha test

Creating a gitaly ha test that covers
the most basic functionality of the
high availability setup.

doc/development/testing_guide/end_to_end/rspec_metadata_tests.md

@@ -14,4 +14,5 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec
 | `:reliable`       | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. |
 | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. |
 | `:runner`         | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. |
+| `:gitaly_ha`      | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are separate containers in a minimal version of [our recommended highly available configuration](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. |
 | `:skip_live_env`  | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. |

qa/qa.rb

@@ -143,6 +143,7 @@ module QA
         autoload :Mattermost, 'qa/scenario/test/integration/mattermost'
         autoload :ObjectStorage, 'qa/scenario/test/integration/object_storage'
         autoload :SMTP, 'qa/scenario/test/integration/smtp'
+        autoload :GitalyHA, 'qa/scenario/test/integration/gitaly_ha'
       end
 
       module Sanity
@@ -449,6 +450,7 @@ module QA
     autoload :Shellout, 'qa/service/shellout'
     autoload :KubernetesCluster, 'qa/service/kubernetes_cluster'
     autoload :Omnibus, 'qa/service/omnibus'
+    autoload :PraefectManager, 'qa/service/praefect_manager'
 
     module ClusterProvider
       autoload :Base, 'qa/service/cluster_provider/base'

qa/qa/scenario/test/integration/gitaly_ha.rb

+# frozen_string_literal: true
+
+module QA
+  module Scenario
+    module Test
+      module Integration
+        class GitalyHA < Test::Instance::All
+          tags :gitaly_ha
+        end
+      end
+    end
+  end
+end

qa/qa/service/praefect_manager.rb

+# frozen_string_literal: true
+
+module QA
+  module Service
+    class PraefectManager
+      include Service::Shellout
+
+      def initialize
+        @praefect = 'praefect'
+        @first_node = 'gitaly1'
+        @second_node = 'gitaly2'
+        @primary_node = @first_node
+        @secondary_node = @second_node
+      end
+
+      def stop_primary_node
+        shell "docker stop #{@primary_node}"
+        @secondary_node, @primary_node = @primary_node, @secondary_node
+      end
+
+      def reset
+        shell "docker start #{@primary_node}"
+        shell "docker start #{@secondary_node}"
+      end
+    end
+  end
+end

qa/qa/specs/features/browser_ui/3_create/gitaly/high_availability_spec.rb

+# frozen_string_literal: true
+
+module QA
+  context 'Create' do
+    context 'Gitaly' do
+      describe 'High Availability', :orchestrated, :gitaly_ha do
+        let(:project) do
+          Resource::Project.fabricate! do |project|
+            project.name = 'gitaly_high_availability'
+          end
+        end
+        let(:initial_file) { 'pushed_to_primary.txt' }
+        let(:final_file) { 'pushed_to_secondary.txt' }
+
+        before do
+          @praefect_manager = Service::PraefectManager.new
+          Flow::Login.sign_in
+        end
+
+        after do
+          @praefect_manager.reset
+        end
+
+        it 'makes sure that automatic failover is happening' do
+          Resource::Repository::ProjectPush.fabricate! do |push|
+            push.project = project
+            push.commit_message = 'pushed to primary gitaly node'
+            push.new_branch = true
+            push.file_name = initial_file
+            push.file_content = "This should exist on both nodes"
+          end
+
+          @praefect_manager.stop_primary_node
+
+          project.visit!
+
+          Page::Project::Show.perform do |show|
+            show.wait_until do
+              show.has_name?(project.name)
+            end
+            expect(show).to have_file(initial_file)
+          end
+
+          Resource::Repository::Commit.fabricate_via_api! do |commit|
+            commit.project = project
+            commit.add_files([
+              {
+                file_path: 'committed_to_primary.txt',
+                content: 'This should exist on both nodes too'
+              }
+            ])
+          end
+
+          project.visit!
+
+          Page::Project::Show.perform do |show|
+            expect(show).to have_file(final_file)
+          end
+        end
+      end
+    end
+  end
+end