Commit 1db40d61 authored by Mark Lapierre's avatar Mark Lapierre

Merge branch 'ml-add-e2e-test-of-gitaly-cluster-backend-node-recovery' into 'master'

Add e2e test of gitaly cluster backend node recovery

See merge request gitlab-org/gitlab!37040
parents 966a6aa7 7b8e5b7d
...@@ -123,6 +123,10 @@ module QA ...@@ -123,6 +123,10 @@ module QA
"#{api_get_path}/runners" "#{api_get_path}/runners"
end end
def api_commits_path
"#{api_get_path}/repository/commits"
end
def api_repository_branches_path def api_repository_branches_path
"#{api_get_path}/repository/branches" "#{api_get_path}/repository/branches"
end end
...@@ -176,6 +180,10 @@ module QA ...@@ -176,6 +180,10 @@ module QA
raise Runtime::API::RepositoryStorageMoves::RepositoryStorageMovesError, 'Timed out while waiting for the repository storage move to finish' raise Runtime::API::RepositoryStorageMoves::RepositoryStorageMovesError, 'Timed out while waiting for the repository storage move to finish'
end end
def commits
parse_body(get(Runtime::API::Request.new(api_client, api_commits_path).url))
end
def import_status def import_status
response = get Runtime::API::Request.new(api_client, "/projects/#{id}/import").url response = get Runtime::API::Request.new(api_client, "/projects/#{id}/import").url
......
This diff is collapsed.
...@@ -33,6 +33,31 @@ module QA ...@@ -33,6 +33,31 @@ module QA
end end
end end
end end
def sql_to_docker_exec_cmd(sql, username, password, database, host, container)
<<~CMD
docker exec --env PGPASSWORD=#{password} #{container} \
bash -c "psql -U #{username} -d #{database} -h #{host} -c \\"#{sql}\\""
CMD
end
def wait_until_shell_command(cmd, **kwargs)
sleep_interval = kwargs.delete(:sleep_interval) || 1
Support::Waiter.wait_until(sleep_interval: sleep_interval, **kwargs) do
shell cmd do |line|
break true if yield line
end
end
end
def wait_until_shell_command_matches(cmd, regex, **kwargs)
wait_until_shell_command(cmd, kwargs) do |line|
QA::Runtime::Logger.debug(line.chomp)
line =~ regex
end
end
end end
end end
end end
# frozen_string_literal: true
module QA
RSpec.describe 'Create' do
context 'Gitaly' do
describe 'Backend node recovery', :orchestrated, :gitaly_ha, :skip_live_env do
let(:praefect_manager) { Service::PraefectManager.new }
let(:project) do
Resource::Project.fabricate! do |project|
project.name = "gitaly_cluster"
project.initialize_with_readme = true
end
end
before do
# Reset the cluster in case previous tests left it in a bad state
praefect_manager.reset_primary_to_original
end
after do
# Leave the cluster in a suitable state for subsequent tests
praefect_manager.reset_primary_to_original
end
it 'recovers from dataloss' do
# Create a new project with a commit and wait for it to replicate
praefect_manager.wait_for_replication(project.id)
# Stop the primary node to trigger failover, and then wait
# for Gitaly to be ready for writes again
praefect_manager.trigger_failover_by_stopping_primary_node
praefect_manager.wait_for_new_primary
praefect_manager.wait_for_health_check_current_primary_node
praefect_manager.wait_for_gitaly_check
# Confirm that we have access to the repo after failover
Support::Waiter.wait_until(retry_on_exception: true, sleep_interval: 5) do
Resource::Repository::Commit.fabricate_via_api! do |commits|
commits.project = project
commits.sha = 'master'
end
end
# Push a commit to the new primary
Resource::Repository::ProjectPush.fabricate! do |push|
push.project = project
push.new_branch = false
push.commit_message = 'pushed after failover'
push.file_name = 'new_file'
push.file_content = 'new file'
end
# Start the old primary node again
praefect_manager.start_primary_node
praefect_manager.wait_for_health_check_current_primary_node
# Confirm dataloss (i.e., inconsistent nodes)
expect(praefect_manager.dataloss?).to be true
expect(praefect_manager.replicated?(project.id)).to be false
# Reconcile nodes to recover from dataloss
praefect_manager.reconcile_nodes
praefect_manager.wait_for_replication(project.id)
# Confirm that both commits are available after reconciliation
expect(project.commits.map { |commit| commit[:message].chomp })
.to include("Initial commit").and include("pushed after failover")
end
end
end
end
end
...@@ -41,8 +41,6 @@ module QA ...@@ -41,8 +41,6 @@ module QA
expect(show).to have_file(initial_file) expect(show).to have_file(initial_file)
end end
praefect_manager.enable_writes
Resource::Repository::Commit.fabricate_via_api! do |commit| Resource::Repository::Commit.fabricate_via_api! do |commit|
commit.project = project commit.project = project
commit.add_files([ commit.add_files([
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment