Skip to content

G
gitlab-ce
Commit 57c7a711 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Merge branch 'mk/move-geo-proxy-endpoint' into 'master' 

Geo: Move /api/v4/internal/geo_proxy to /api/v4/geo/proxy

See merge request gitlab-org/gitlab!63343
parents 3cbea463 7a8fe83d
Hide whitespace changes
Inline Side-by-side
Showing with 345 additions and 310 deletions
ee/lib/api/geo.rb deleted 100644 → 0
View file @ 3cbea463
# frozen_string_literal: true
require 'base64'
module API
class Geo < ::API::Base
feature_category :geo_replication
resource :geo do
helpers do
def sanitized_node_status_params
valid_attributes = GeoNodeStatus.attribute_names - GeoNodeStatus::RESOURCE_STATUS_FIELDS - ['id']
sanitized_params = params.slice(*valid_attributes)
# sanitize status field
sanitized_params['status'] = sanitized_params['status'].slice(*GeoNodeStatus::RESOURCE_STATUS_FIELDS) if sanitized_params['status']
sanitized_params
end
# Check if a Geo request is legit or fail the flow
#
# @param [Hash] attributes to be matched against JWT
def authorize_geo_transfer!(**attributes)
unauthorized! unless geo_jwt_decoder.valid_attributes?(**attributes)
end
end
params do
requires :replicable_name, type: String, desc: 'Replicable name (eg. package_file)'
requires :replicable_id, type: Integer, desc: 'The replicable ID that needs to be transferred'
end
get 'retrieve/:replicable_name/:replicable_id' do
check_gitlab_geo_request_ip!
params_sym = params.symbolize_keys
authorize_geo_transfer!(**params_sym)
decoded_params = geo_jwt_decoder.decode
service = ::Geo::BlobUploadService.new(**params_sym, decoded_params: decoded_params)
response = service.execute
if response[:code] == :ok
file = response[:file]
present_carrierwave_file!(file)
else
error! response, response.delete(:code)
end
end
# Verify the GitLab Geo transfer request is valid
# All transfers use the Authorization header to pass a JWT
# payload.
#
# For LFS objects, validate the object ID exists in the DB
# and that the object ID matches the requested ID. This is
# a sanity check against some malicious client requesting
# a random file path.
params do
requires :type, type: String, desc: 'File transfer type (e.g. lfs)'
requires :id, type: Integer, desc: 'The DB ID of the file'
end
get 'transfers/:type/:id' do
check_gitlab_geo_request_ip!
authorize_geo_transfer!(file_type: params[:type], file_id: params[:id])
decoded_params = geo_jwt_decoder.decode
service = ::Geo::FileUploadService.new(params, decoded_params)
response = service.execute
if response[:code] == :ok
file = response[:file]
present_carrierwave_file!(file)
else
error! response, response.delete(:code)
end
end
# Post current node information to primary (e.g. health, repos synced, repos failed, etc.)
#
# Example request:
# POST /geo/status
post 'status' do
check_gitlab_geo_request_ip!
authenticate_by_gitlab_geo_node_token!
db_status = GeoNode.find(params[:geo_node_id]).find_or_build_status
unless db_status.update(sanitized_node_status_params.merge(last_successful_status_check_at: Time.now.utc))
render_validation_error!(db_status)
end
end
# git over SSH secondary endpoints -> primary related proxying logic
#
resource 'proxy_git_ssh' do
format :json
# For git clone/pull
# Responsible for making HTTP GET /repo.git/info/refs?service=git-upload-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
end
post 'info_refs_upload_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = Gitlab::Geo::GitSSHProxy.new(params['data']).info_refs_upload_pack
status(response.code)
response.body
end
# Responsible for making HTTP POST /repo.git/git-upload-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
requires :output, type: String, desc: 'Output from git-upload-pack'
end
post 'upload_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = Gitlab::Geo::GitSSHProxy.new(params['data']).upload_pack(params['output'])
status(response.code)
response.body
end
# For git push
# Responsible for making HTTP GET /repo.git/info/refs?service=git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
end
post 'info_refs_receive_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = Gitlab::Geo::GitSSHProxy.new(params['data']).info_refs_receive_pack
status(response.code)
response.body
end
# Responsible for making HTTP POST /repo.git/git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
requires :output, type: String, desc: 'Output from git-receive-pack'
end
post 'receive_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = Gitlab::Geo::GitSSHProxy.new(params['data']).receive_pack(params['output'])
status(response.code)
response.body
end
end
end
end
end
ee/lib/ee/api/api.rb
View file @ 57c7a711
......@@ -20,7 +20,6 @@ module EE
mount ::API::Epics
mount ::API::ElasticsearchIndexedNamespaces
mount ::API::Experiments
mount ::API::Geo
mount ::API::GeoReplication
mount ::API::GeoNodes
mount ::API::Ldap
......
ee/lib/ee/api/geo.rb 0 → 100644
View file @ 57c7a711
# frozen_string_literal: true
require 'base64'
module EE
module API
module Geo
extend ActiveSupport::Concern
prepended do
helpers do
extend ::Gitlab::Utils::Override
def sanitized_node_status_params
valid_attributes = GeoNodeStatus.attribute_names - GeoNodeStatus::RESOURCE_STATUS_FIELDS - ['id']
sanitized_params = params.slice(*valid_attributes)
# sanitize status field
sanitized_params['status'] = sanitized_params['status'].slice(*GeoNodeStatus::RESOURCE_STATUS_FIELDS) if sanitized_params['status']
sanitized_params
end
# Check if a Geo request is legit or fail the flow
#
# @param [Hash] attributes to be matched against JWT
def authorize_geo_transfer!(**attributes)
unauthorized! unless geo_jwt_decoder.valid_attributes?(**attributes)
end
override :geo_proxy_response
def geo_proxy_response
# The methods used here (or their underlying methods) are cached
# for:
#
# * 1 minute in memory
# * 2 minutes in Redis
#
# The cached values are invalidated when changed.
#
if ::Feature.enabled?(:geo_secondary_proxy, default_enabled: :yaml) && ::Gitlab::Geo.secondary_with_primary?
{ geo_proxy_url: ::Gitlab::Geo.primary_node.internal_url }
else
super
end
end
end
resource :geo do
params do
requires :replicable_name, type: String, desc: 'Replicable name (eg. package_file)'
requires :replicable_id, type: Integer, desc: 'The replicable ID that needs to be transferred'
end
get 'retrieve/:replicable_name/:replicable_id' do
check_gitlab_geo_request_ip!
params_sym = params.symbolize_keys
authorize_geo_transfer!(**params_sym)
decoded_params = geo_jwt_decoder.decode
service = ::Geo::BlobUploadService.new(**params_sym, decoded_params: decoded_params)
response = service.execute
if response[:code] == :ok
file = response[:file]
present_carrierwave_file!(file)
else
error! response, response.delete(:code)
end
end
# Verify the GitLab Geo transfer request is valid
# All transfers use the Authorization header to pass a JWT
# payload.
#
# For LFS objects, validate the object ID exists in the DB
# and that the object ID matches the requested ID. This is
# a sanity check against some malicious client requesting
# a random file path.
params do
requires :type, type: String, desc: 'File transfer type (e.g. lfs)'
requires :id, type: Integer, desc: 'The DB ID of the file'
end
get 'transfers/:type/:id' do
check_gitlab_geo_request_ip!
authorize_geo_transfer!(file_type: params[:type], file_id: params[:id])
decoded_params = geo_jwt_decoder.decode
service = ::Geo::FileUploadService.new(params, decoded_params)
response = service.execute
if response[:code] == :ok
file = response[:file]
present_carrierwave_file!(file)
else
error! response, response.delete(:code)
end
end
# Post current node information to primary (e.g. health, repos synced, repos failed, etc.)
#
# Example request:
# POST /geo/status
post 'status' do
check_gitlab_geo_request_ip!
authenticate_by_gitlab_geo_node_token!
db_status = GeoNode.find(params[:geo_node_id]).find_or_build_status
unless db_status.update(sanitized_node_status_params.merge(last_successful_status_check_at: Time.now.utc))
render_validation_error!(db_status)
end
end
# git over SSH secondary endpoints -> primary related proxying logic
#
resource 'proxy_git_ssh' do
format :json
# For git clone/pull
# Responsible for making HTTP GET /repo.git/info/refs?service=git-upload-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
end
post 'info_refs_upload_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = ::Gitlab::Geo::GitSSHProxy.new(params['data']).info_refs_upload_pack
status(response.code)
response.body
end
# Responsible for making HTTP POST /repo.git/git-upload-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
requires :output, type: String, desc: 'Output from git-upload-pack'
end
post 'upload_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = ::Gitlab::Geo::GitSSHProxy.new(params['data']).upload_pack(params['output'])
status(response.code)
response.body
end
# For git push
# Responsible for making HTTP GET /repo.git/info/refs?service=git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
end
post 'info_refs_receive_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = ::Gitlab::Geo::GitSSHProxy.new(params['data']).info_refs_receive_pack
status(response.code)
response.body
end
# Responsible for making HTTP POST /repo.git/git-receive-pack
# request *from* secondary gitlab-shell to primary
#
params do
requires :secret_token, type: String
requires :data, type: Hash do
requires :gl_id, type: String
requires :primary_repo, type: String
end
requires :output, type: String, desc: 'Output from git-receive-pack'
end
post 'receive_pack' do
authenticate_by_gitlab_shell_token!
params.delete(:secret_token)
response = ::Gitlab::Geo::GitSSHProxy.new(params['data']).receive_pack(params['output'])
status(response.code)
response.body
end
end
end
end
end
end
end
ee/lib/ee/api/internal/base.rb
View file @ 57c7a711
......@@ -49,21 +49,6 @@ module EE
{ success: false, message: 'Invalid OTP' }
end
end
override :geo_proxy
def geo_proxy
# The methods used here (or their underlying methods) are cached
# for:
#
# * 1 minute in memory
# * 2 minutes in Redis
#
if ::Feature.enabled?(:geo_secondary_proxy, default_enabled: :yaml) && ::Gitlab::Geo.secondary_with_primary?
{ geo_proxy_url: ::Gitlab::Geo.primary_node.internal_url }
else
super
end
end
end
end
end
......
ee/spec/requests/api/geo_spec.rb
View file @ 57c7a711
......@@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe API::Geo do
include TermsHelper
include ApiHelpers
include WorkhorseHelpers
include ::EE::GeoHelpers
let_it_be(:admin) { create(:admin) }
......@@ -644,4 +645,81 @@ RSpec.describe API::Geo do
end
end
end
describe 'GET /geo/proxy' do
subject { get api('/geo/proxy'), headers: workhorse_headers }
include_context 'workhorse headers'
context 'with valid auth' do
context 'when Geo is not being used' do
it 'returns empty data' do
allow(::Gitlab::Geo).to receive(:enabled?).and_return(false)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
context 'when this is a primary site' do
it 'returns empty data' do
stub_current_geo_node(primary_node)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
context 'when this is a secondary site' do
before do
stub_current_geo_node(secondary_node)
end
context 'when a primary exists' do
it 'returns the primary internal URL' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['geo_proxy_url']).to match(primary_node.internal_url)
end
end
context 'when a primary does not exist' do
it 'returns empty data' do
allow(::Gitlab::Geo).to receive(:primary_node_configured?).and_return(false)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
end
context 'when geo_secondary_proxy feature flag is disabled' do
before do
stub_feature_flags(geo_secondary_proxy: false)
end
it 'returns empty data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
end
it 'rejects requests that bypassed gitlab-workhorse' do
workhorse_headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
ee/spec/requests/api/internal/base_spec.rb
View file @ 57c7a711
......@@ -474,81 +474,4 @@ RSpec.describe API::Internal::Base do
end
end
end
describe 'GET /internal/geo_proxy' do
subject { get api('/internal/geo_proxy'), params: { secret_token: secret_token } }
context 'with valid auth' do
context 'when Geo is not being used' do
it 'returns empty data' do
allow(::Gitlab::Geo).to receive(:enabled?).and_return(false)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
context 'when this is a primary site' do
it 'returns empty data' do
stub_current_geo_node(primary_node)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
context 'when this is a secondary site' do
before do
stub_current_geo_node(secondary_node)
end
context 'when a primary exists' do
it 'returns the primary internal URL' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['geo_proxy_url']).to match(primary_node.internal_url)
end
end
context 'when a primary does not exist' do
it 'returns empty data' do
allow(::Gitlab::Geo).to receive(:primary_node_configured?).and_return(false)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
end
context 'when geo_secondary_proxy feature flag is disabled' do
before do
stub_feature_flags(geo_secondary_proxy: false)
end
it 'returns empty data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
end
context 'with invalid auth' do
let(:secret_token) { 'invalid_token' }
it 'returns unauthorized' do
subject
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
end
end
lib/api/api.rb
View file @ 57c7a711
......@@ -172,6 +172,7 @@ module API
mount ::API::Features
mount ::API::Files
mount ::API::FreezePeriods
mount ::API::Geo
mount ::API::GroupAvatar
mount ::API::GroupBoards
mount ::API::GroupClusters
......
lib/api/geo.rb 0 → 100644
View file @ 57c7a711
# frozen_string_literal: true
module API
class Geo < ::API::Base
feature_category :geo_replication
helpers do
# Overridden in EE
def geo_proxy_response
{}
end
end
resource :geo do
# Workhorse calls this to determine if it is a Geo site that should proxy
# requests. Workhorse doesn't know if it's in a FOSS/EE context.
get '/proxy' do
require_gitlab_workhorse!
status :ok
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
geo_proxy_response
end
end
end
end
API::Geo.prepend_mod
lib/api/internal/base.rb
View file @ 57c7a711
......@@ -124,11 +124,6 @@ module API
yield
end
end
# Overridden in EE
def geo_proxy
{}
end
end
namespace 'internal' do
......@@ -320,12 +315,6 @@ module API
two_factor_otp_check
end
# Workhorse calls this to determine if it is a Geo secondary site
# that should proxy requests. FOSS can quickly return empty data.
get '/geo_proxy', feature_category: :geo_replication do
geo_proxy
end
end
end
end
......
spec/requests/api/geo_spec.rb 0 → 100644
View file @ 57c7a711
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Geo do
include WorkhorseHelpers
describe 'GET /geo/proxy' do
subject { get api('/geo/proxy'), headers: workhorse_headers }
include_context 'workhorse headers'
context 'with valid auth' do
it 'returns empty data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
it 'rejects requests that bypassed gitlab-workhorse' do
workhorse_headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
spec/requests/api/internal/base_spec.rb
View file @ 57c7a711
......@@ -1378,29 +1378,6 @@ RSpec.describe API::Internal::Base do
end
end
describe 'GET /internal/geo_proxy' do
subject { get api('/internal/geo_proxy'), params: { secret_token: secret_token } }
context 'with valid auth' do
it 'returns empty data' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to be_empty
end
end
context 'with invalid auth' do
let(:secret_token) { 'invalid_token' }
it 'returns unauthorized' do
subject
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
end
def lfs_auth_project(project)
post(
api("/internal/lfs_authenticate"),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7