Commit 482577c6 authored by Stan Hu's avatar Stan Hu

Merge branch 'ash.mckenzie/geo-git-push-ssh-proxy' into 'master'

SSH git push to secondary -> proxy to Primary

See merge request gitlab-org/gitlab-ee!6456
parents 58d7871a a9b10abc
......@@ -202,9 +202,8 @@ Read how to [replicate the Container Registry][docker-registry].
extra limitations may be in place.
- Pushing code to a secondary redirects the request to the primary instead of handling it directly [gitlab-ee#1381](https://gitlab.com/gitlab-org/gitlab-ee/issues/1381):
* Only push via HTTP is currently supported
* Git LFS is supported
* Pushing via SSH is currently not supported: [gitlab-ee#5387](https://gitlab.com/gitlab-org/gitlab-ee/issues/5387)
* Push via HTTP and SSH supported
* Git LFS also supported
- The primary node has to be online for OAuth login to happen (existing sessions and Git are not affected)
- The installation takes multiple manual steps that together can take about an hour depending on circumstances; we are
working on improving this experience, see [gitlab-org/omnibus-gitlab#2978] for details.
......
---
title: 'Geo: SSH git push to secondary -> proxy to Primary'
merge_request: 6456
author:
type: added
require 'base64'
module API
class Geo < Grape::API
resource :geo do
......@@ -40,6 +42,51 @@ module API
render_validation_error!(db_status)
end
end
# git push over SSH secondary -> primary related proxying logic
#
resource 'proxy_git_push_ssh' do
format :json
# Responsible for making HTTP GET /repo.git/info/refs?service=git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
end
post 'info_refs' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
resp = Gitlab::Geo::GitPushSSHProxy.new(params['data']).info_refs
status(resp.code.to_i)
{ status: true, message: nil, result: Base64.encode64(resp.body.to_s) }
end
# Responsible for making HTTP POST /repo.git/git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
requires :output, type: String, desc: 'Output from git-receive-pack'
end
post 'push' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
resp = Gitlab::Geo::GitPushSSHProxy.new(params['data']).push(Base64.decode64(params['output']))
status(resp.code.to_i)
{ status: true, message: nil, result: Base64.encode64(resp.body.to_s) }
end
end
end
end
end
......@@ -3,9 +3,19 @@ module EE
module GeoGitAccess
include ::Gitlab::ConfigHelper
include ::EE::GitlabRoutingHelper
include GrapePathHelpers::NamedRouteMatcher
extend ::Gitlab::Utils::Override
GEO_SERVER_DOCS_URL = 'https://docs.gitlab.com/ee/administration/geo/replication/using_a_geo_server.html'.freeze
override :check_custom_action
def check_custom_action(cmd)
custom_action = custom_action_for(cmd)
return custom_action if custom_action
super
end
protected
def project_or_wiki
......@@ -14,6 +24,27 @@ module EE
private
def custom_action_for?(cmd)
return unless receive_pack?(cmd)
return unless ::Gitlab::Database.read_only?
::Gitlab::Geo.secondary_with_primary?
end
def custom_action_for(cmd)
return unless custom_action_for?(cmd)
payload = {
'action' => 'geo_proxy_to_primary',
'data' => {
'api_endpoints' => [api_v4_geo_proxy_git_push_ssh_info_refs_path, api_v4_geo_proxy_git_push_ssh_push_path],
'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki)
}
}
::Gitlab::GitAccessResult::CustomAction.new(payload, 'Attempting to proxy to primary.')
end
def push_to_read_only_message
message = super
......
......@@ -17,7 +17,7 @@ module EE
override :whitelisted_routes
def whitelisted_routes
super || geo_node_update_route
super || geo_node_update_route || geo_proxy_git_push_ssh_route
end
def geo_node_update_route
......@@ -33,6 +33,14 @@ module EE
WHITELISTED_GEO_ROUTES_TRACKING_DB[controller]&.include?(action)
end
end
def geo_proxy_git_push_ssh_route
routes = ::Gitlab::Middleware::ReadOnly::API_VERSIONS.map do |version|
["/api/v#{version}/geo/proxy_git_push_ssh/info_refs",
"/api/v#{version}/geo/proxy_git_push_ssh/push"]
end
routes.flatten.include?(request.path)
end
end
end
end
......
# frozen_string_literal: true
module Gitlab
module Geo
class GitPushSSHProxy
HTTP_READ_TIMEOUT = 10
HTTP_SUCCESS_CODE = '200'.freeze
MustBeASecondaryNode = Class.new(StandardError)
def initialize(data)
@data = data
end
def info_refs
ensure_secondary!
url = "#{primary_repo}/info/refs?service=git-receive-pack"
headers = {
'Content-Type' => 'application/x-git-upload-pack-request'
}
resp = get(url, headers)
return resp unless resp.code == HTTP_SUCCESS_CODE
resp.body = remove_http_service_fragment_from(resp.body)
resp
end
def push(info_refs_response)
ensure_secondary!
url = "#{primary_repo}/git-receive-pack"
headers = {
'Content-Type' => 'application/x-git-receive-pack-request',
'Accept' => 'application/x-git-receive-pack-result'
}
post(url, info_refs_response, headers)
end
private
attr_reader :data
def primary_repo
@primary_repo ||= data['primary_repo']
end
def gl_id
@gl_id ||= data['gl_id']
end
def base_headers
@base_headers ||= {
'Geo-GL-Id' => gl_id,
'Authorization' => Gitlab::Geo::BaseRequest.new.authorization
}
end
def get(url, headers)
request(url, Net::HTTP::Get, headers)
end
def post(url, body, headers)
request(url, Net::HTTP::Post, headers, body: body)
end
def request(url, klass, headers, body: nil)
headers = base_headers.merge(headers)
uri = URI.parse(url)
req = klass.new(uri, headers)
req.body = body if body
http = Net::HTTP.new(uri.hostname, uri.port)
http.read_timeout = HTTP_READ_TIMEOUT
http.use_ssl = true if uri.is_a?(URI::HTTPS)
http.start { http.request(req) }
end
def remove_http_service_fragment_from(body)
# HTTP(S) and SSH responses are very similar, except for the fragment below.
# As we're performing a git HTTP(S) request here, we'll get a HTTP(s)
# suitable git response. However, we're executing in the context of an
# SSH session so we need to make the response suitable for what git over
# SSH expects.
#
# See Downloading Data > HTTP(S) section at:
# https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
body.gsub(/\A001f# service=git-receive-pack\n0000/, '')
end
def ensure_secondary!
raise MustBeASecondaryNode, 'Node is not a secondary' unless Gitlab::Geo.secondary_with_primary?
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Geo::GitPushSSHProxy, :geo do
include ::EE::GeoHelpers
set(:primary_node) { create(:geo_node, :primary) }
set(:secondary_node) { create(:geo_node) }
let(:current_node) { nil }
let(:project) { create(:project, :repository) }
let(:user) { project.creator }
let(:key) { create(:key, user: user) }
let(:base_request) { double(Gitlab::Geo::BaseRequest.new.authorization) }
let(:info_refs_body_short) do
"008f43ba78b7912f7bf7ef1d7c3b8a0e5ae14a759dfa refs/heads/masterreport-status delete-refs side-band-64k quiet atomic ofs-delta agent=git/2.18.0
0000"
end
let(:base_headers) do
{
'Geo-GL-Id' => "key-#{key.id}",
'Authorization' => 'secret'
}
end
let(:data) do
{
'gl_id' => "key-#{key.id}",
'primary_repo' => "#{primary_node.url}#{project.repository.full_path}.git"
}
end
subject { described_class.new(data) }
before do
stub_current_geo_node(current_node)
allow(Gitlab::Geo::BaseRequest).to receive(:new).and_return(base_request)
allow(base_request).to receive(:authorization).and_return('secret')
end
describe '#info_refs' do
context 'against primary node' do
let(:current_node) { primary_node }
it 'raises an exception' do
expect do
subject.info_refs
end.to raise_error(described_class::MustBeASecondaryNode)
end
end
context 'against secondary node' do
let(:current_node) { secondary_node }
let(:full_info_refs_url) { "#{primary_node.url}#{project.full_path}.git/info/refs?service=git-receive-pack" }
let(:info_refs_headers) { base_headers.merge('Content-Type' => 'application/x-git-upload-pack-request') }
let(:info_refs_http_body_full) do
"001f# service=git-receive-pack
0000#{info_refs_body_short}"
end
before do
stub_request(:get, full_info_refs_url).to_return(status: 200, body: info_refs_http_body_full, headers: info_refs_headers)
end
it 'returns a Net::HTTPOK' do
expect(subject.info_refs).to be_a(Net::HTTPOK)
end
it 'returns a modified body' do
expect(subject.info_refs.body).to eql(info_refs_body_short)
end
end
end
describe '#push' do
context 'against primary node' do
let(:current_node) { primary_node }
it 'raises an exception' do
expect do
subject.push(info_refs_body_short)
end.to raise_error(described_class::MustBeASecondaryNode)
end
end
context 'against secondary node' do
let(:current_node) { secondary_node }
let(:full_git_receive_pack_url) { "#{primary_node.url}#{project.full_path}.git/git-receive-pack" }
let(:push_headers) do
base_headers.merge(
'Content-Type' => 'application/x-git-receive-pack-request',
'Accept' => 'application/x-git-receive-pack-result'
)
end
before do
stub_request(:post, full_git_receive_pack_url).to_return(status: 201, body: info_refs_body_short, headers: push_headers)
end
it 'returns a Net::HTTPCreated' do
expect(subject.push(info_refs_body_short)).to be_a(Net::HTTPCreated)
end
end
end
end
......@@ -18,25 +18,9 @@ describe Gitlab::GitAccess do
allow(Gitlab::Database).to receive(:read_only?) { true }
end
it 'denies push access' do
project.add_maintainer(user)
let(:primary_repo_url) { "https://localhost:3000/gitlab/#{project.full_path}.git" }
expect { push_changes }.to raise_unauthorized("You can't push code to a read-only GitLab instance.")
end
it 'denies push access with primary present' do
error_message = "You can't push code to a read-only GitLab instance."\
"\nPlease use the primary node URL instead: https://localhost:3000/gitlab/#{project.full_path}.git.
For more information: #{EE::Gitlab::GeoGitAccess::GEO_SERVER_DOCS_URL}"
primary_node = create(:geo_node, :primary, url: 'https://localhost:3000/gitlab')
allow(Gitlab::Geo).to receive(:primary).and_return(primary_node)
allow(Gitlab::Geo).to receive(:secondary_with_primary?).and_return(true)
project.add_maintainer(user)
expect { push_changes }.to raise_unauthorized(error_message)
end
it_behaves_like 'a read-only GitLab instance'
end
describe "push_rule_check" do
......
require 'spec_helper'
describe Gitlab::GitAccessWiki do
let(:access) { described_class.new(user, project, 'web', authentication_abilities: authentication_abilities, redirected_path: redirected_path) }
let(:project) { create(:project, :repository) }
let(:user) { create(:user) }
let(:project) { create(:project, :repository) }
let(:changes) { ['6f6d7e7ed 570e7b2ab refs/heads/master'] }
let(:authentication_abilities) { %i[read_project download_code push_code] }
let(:redirected_path) { nil }
let(:authentication_abilities) do
[
:read_project,
:download_code,
:push_code
]
end
let(:access) { described_class.new(user, project, 'web', authentication_abilities: authentication_abilities, redirected_path: redirected_path) }
context "when in a read-only GitLab instance" do
subject { access.check('git-receive-pack', changes) }
......@@ -22,26 +17,9 @@ describe Gitlab::GitAccessWiki do
allow(Gitlab::Database).to receive(:read_only?) { true }
end
it 'denies push access' do
project.add_maintainer(user)
expect { subject }.to raise_unauthorized("You can't push code to a read-only GitLab instance.")
end
it 'denies push access with primary present' do
error_message = "You can't push code to a read-only GitLab instance.
Please use the primary node URL instead: "\
"https://localhost:3000/gitlab/#{project.full_path}.wiki.git.
For more information: #{EE::Gitlab::GeoGitAccess::GEO_SERVER_DOCS_URL}"
primary_node = create(:geo_node, :primary, url: 'https://localhost:3000/gitlab')
allow(Gitlab::Geo).to receive(:primary).and_return(primary_node)
allow(Gitlab::Geo).to receive(:secondary_with_primary?).and_return(true)
project.add_maintainer(user)
let(:primary_repo_url) { "https://localhost:3000/gitlab/#{project.full_path}.wiki.git" }
expect { subject }.to raise_unauthorized(error_message)
end
it_behaves_like 'a read-only GitLab instance'
end
context 'when wiki is disabled' do
......@@ -58,6 +36,10 @@ For more information: #{EE::Gitlab::GeoGitAccess::GEO_SERVER_DOCS_URL}"
private
def push_changes(changes = '_any')
access.check('git-receive-pack', changes)
end
def raise_unauthorized(message)
raise_error(Gitlab::GitAccess::UnauthorizedError, message)
end
......
......@@ -25,7 +25,7 @@ describe API::Geo do
end
end
describe '/geo/transfers' do
describe 'GET /geo/transfers' do
before do
stub_current_geo_node(secondary_node)
end
......@@ -287,4 +287,120 @@ describe API::Geo do
it_behaves_like 'with terms enforced'
end
end
describe '/geo/proxy_git_push_ssh' do
let(:secret_token) { Gitlab::Shell.secret_token }
let(:data) { { primary_repo: 'http://localhost:3001/testuser/repo.git', gl_id: 'key-1', gl_username: 'testuser' } }
before do
stub_current_geo_node(secondary_node)
end
describe 'POST /geo/proxy_git_push_ssh/info_refs' do
context 'with all required params missing' do
it 'responds with 400' do
post api('/geo/proxy_git_push_ssh/info_refs'), nil
expect(response).to have_gitlab_http_status(400)
expect(json_response['error']).to eql('secret_token is missing, data is missing, data[gl_id] is missing, data[primary_repo] is missing')
end
end
context 'with all required params' do
let(:git_push_ssh_proxy) { double(Gitlab::Geo::GitPushSSHProxy) }
before do
allow(Gitlab::Geo::GitPushSSHProxy).to receive(:new).with(data).and_return(git_push_ssh_proxy)
end
context 'with an invalid secret_token' do
it 'responds with 401' do
post(api('/geo/proxy_git_push_ssh/info_refs'), { secret_token: 'invalid', data: data })
expect(response).to have_gitlab_http_status(401)
expect(json_response['error']).to be_nil
end
end
context 'where an exception occurs' do
it 'responds with 500' do
expect(git_push_ssh_proxy).to receive(:info_refs).and_raise('deliberate exception raised')
post api('/geo/proxy_git_push_ssh/info_refs'), { secret_token: secret_token, data: data }
expect(response).to have_gitlab_http_status(500)
expect(json_response['message']).to include('RuntimeError (deliberate exception raised)')
expect(json_response['result']).to be_nil
end
end
context 'with a valid secret token' do
let(:http_response) { double(Net::HTTPResponse, code: 200, body: 'something here') }
it 'responds with 200' do
expect(git_push_ssh_proxy).to receive(:info_refs).and_return(http_response)
post api('/geo/proxy_git_push_ssh/info_refs'), { secret_token: secret_token, data: data }
expect(response).to have_gitlab_http_status(200)
expect(Base64.decode64(json_response['result'])).to eql('something here')
end
end
end
end
describe 'POST /geo/proxy_git_push_ssh/push' do
context 'with all required params missing' do
it 'responds with 400' do
post api('/geo/proxy_git_push_ssh/push'), nil
expect(response).to have_gitlab_http_status(400)
expect(json_response['error']).to eql('secret_token is missing, data is missing, data[gl_id] is missing, data[primary_repo] is missing, output is missing')
end
end
context 'with all required params' do
let(:text) { 'output text' }
let(:output) { Base64.encode64(text) }
let(:git_push_ssh_proxy) { double(Gitlab::Geo::GitPushSSHProxy) }
before do
allow(Gitlab::Geo::GitPushSSHProxy).to receive(:new).with(data).and_return(git_push_ssh_proxy)
end
context 'with an invalid secret_token' do
it 'responds with 401' do
post(api('/geo/proxy_git_push_ssh/push'), { secret_token: 'invalid', data: data, output: output })
expect(response).to have_gitlab_http_status(401)
expect(json_response['error']).to be_nil
end
end
context 'where an exception occurs' do
it 'responds with 500' do
expect(git_push_ssh_proxy).to receive(:push).and_raise('deliberate exception raised')
post api('/geo/proxy_git_push_ssh/push'), { secret_token: secret_token, data: data, output: output }
expect(response).to have_gitlab_http_status(500)
expect(json_response['message']).to include('RuntimeError (deliberate exception raised)')
expect(json_response['result']).to be_nil
end
end
context 'with a valid secret token' do
let(:http_response) { double(Net::HTTPResponse, code: 201, body: 'something here') }
it 'responds with 201' do
expect(git_push_ssh_proxy).to receive(:push).with(text).and_return(http_response)
post api('/geo/proxy_git_push_ssh/push'), { secret_token: secret_token, data: data, output: output }
expect(response).to have_gitlab_http_status(201)
expect(Base64.decode64(json_response['result'])).to eql('something here')
end
end
end
end
end
end
# frozen_string_literal: true
shared_examples 'a read-only GitLab instance' do
it 'denies push access' do
project.add_maintainer(user)
expect { push_changes }.to raise_unauthorized("You can't push code to a read-only GitLab instance.")
end
context 'for a Geo setup' do
before do
primary_node = create(:geo_node, :primary, url: 'https://localhost:3000/gitlab')
allow(Gitlab::Geo).to receive(:primary).and_return(primary_node)
allow(Gitlab::Geo).to receive(:secondary_with_primary?).and_return(secondary_with_primary)
end
context 'that is incorrectly setup' do
let(:secondary_with_primary) { false }
let(:error_message) { "You can't push code to a read-only GitLab instance." }
it 'denies push access with primary present' do
project.add_maintainer(user)
expect { push_changes }.to raise_unauthorized(error_message)
end
end
context 'that is correctly setup' do
let(:secondary_with_primary) { true }
let(:payload) do
{
'action' => 'geo_proxy_to_primary',
'data' => {
'api_endpoints' => %w{/api/v4/geo/proxy_git_push_ssh/info_refs /api/v4/geo/proxy_git_push_ssh/push},
'primary_repo' => primary_repo_url
}
}
end
it 'attempts to proxy to the primary' do
project.add_maintainer(user)
expect(push_changes).to be_a(Gitlab::GitAccessResult::CustomAction)
expect(push_changes.message).to eql('Attempting to proxy to primary.')
expect(push_changes.payload).to eql(payload)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment