Merge branch 'allow-raw-blobs-to-be-served-from-external-storage' into 'master'

Allow raw blobs to be served from an external storage

See merge request gitlab-org/gitlab!20936

app/controllers/projects/raw_controller.rb

@@ -4,11 +4,15 @@
 class Projects::RawController < Projects::ApplicationController
   include ExtractsPath
   include SendsBlob
+  include StaticObjectExternalStorage
+
+  prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:blob) }
 
   before_action :require_non_empty_project
   before_action :assign_ref_vars
   before_action :authorize_download_code!
-  before_action :show_rate_limit, only: [:show]
+  before_action :show_rate_limit, only: [:show], unless: :external_storage_request?
+  before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled?
 
   def show
     @blob = @repository.blob_at(@commit.id, @path)

app/helpers/blob_helper.rb

@@ -215,14 +215,29 @@ module BlobHelper
     return if blob.binary? || blob.stored_externally?
 
     title = _('Open raw')
-    link_to icon('file-code-o'), blob_raw_path, class: 'btn btn-sm has-tooltip', target: '_blank', rel: 'noopener noreferrer', title: title, data: { container: 'body' }
+    link_to sprite_icon('doc-code'),
+      external_storage_url_or_path(blob_raw_path),
+      class: 'btn btn-sm has-tooltip',
+      target: '_blank',
+      rel: 'noopener noreferrer',
+      aria: { label: title },
+      title: title,
+      data: { container: 'body' }
   end
 
   def download_blob_button(blob)
     return if blob.empty?
 
     title = _('Download')
-    link_to sprite_icon('download'), blob_raw_path(inline: false), download: @path, class: 'btn btn-sm has-tooltip', target: '_blank', rel: 'noopener noreferrer', title: title, data: { container: 'body' }
+    link_to sprite_icon('download'),
+      external_storage_url_or_path(blob_raw_path(inline: false)),
+      download: @path,
+      class: 'btn btn-sm has-tooltip',
+      target: '_blank',
+      rel: 'noopener noreferrer',
+      aria: { label: title },
+      title: title,
+      data: { container: 'body' }
   end
 
   def blob_render_error_reason(viewer)

changelogs/unreleased/allow-raw-blobs-to-be-served-from-external-storage.yml

+---
+title: Allow raw blobs to be served from an external storage
+merge_request: 20936
+author:
+type: added

lib/gitlab/auth/user_auth_finders.rb

@@ -169,6 +169,8 @@ module Gitlab
         case request_format
         when :archive
           archive_request?
+        when :blob
+          blob_request?
         else
           false
         end
@@ -189,6 +191,10 @@ module Gitlab
       def archive_request?
         current_request.path.include?('/-/archive/')
       end
+
+      def blob_request?
+        current_request.path.include?('/raw/')
+      end
     end
   end
 end

spec/controllers/projects/raw_controller_spec.rb

@@ -77,6 +77,24 @@ describe Projects::RawController do
         execute_raw_requests(requests: 6, project: project, file_path: file_path)
       end
 
+      context 'when receiving an external storage request' do
+        let(:token) { 'letmein' }
+
+        before do
+          stub_application_setting(
+            static_objects_external_storage_url: 'https://cdn.gitlab.com',
+            static_objects_external_storage_auth_token: token
+          )
+        end
+
+        it 'does not prevent from accessing the raw file' do
+          request.headers['X-Gitlab-External-Storage-Token'] = token
+          execute_raw_requests(requests: 6, project: project, file_path: file_path)
+
+          expect(response).to have_gitlab_http_status(200)
+        end
+      end
+
       context 'when the request uses a different version of a commit' do
         it 'prevents from accessing the raw file' do
           # 3 times with the normal sha
@@ -131,15 +149,74 @@ describe Projects::RawController do
         end
       end
     end
+
+    context 'as a sessionless user' do
+      let_it_be(:project) { create(:project, :private, :repository) }
+      let_it_be(:user) { create(:user, static_object_token: 'very-secure-token') }
+      let_it_be(:file_path) { 'master/README.md' }
+
+      before do
+        project.add_developer(user)
+      end
+
+      context 'when no token is provided' do
+        it 'redirects to sign in page' do
+          execute_raw_requests(requests: 1, project: project, file_path: file_path)
+
+          expect(response).to have_gitlab_http_status(302)
+          expect(response.location).to end_with('/users/sign_in')
+        end
+      end
+
+      context 'when a token param is present' do
+        context 'when token is correct' do
+          it 'calls the action normally' do
+            execute_raw_requests(requests: 1, project: project, file_path: file_path, token: user.static_object_token)
+
+            expect(response).to have_gitlab_http_status(200)
+          end
+        end
+
+        context 'when token is incorrect' do
+          it 'redirects to sign in page' do
+            execute_raw_requests(requests: 1, project: project, file_path: file_path, token: 'foobar')
+
+            expect(response).to have_gitlab_http_status(302)
+            expect(response.location).to end_with('/users/sign_in')
+          end
+        end
+      end
+
+      context 'when a token header is present' do
+        context 'when token is correct' do
+          it 'calls the action normally' do
+            request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token
+            execute_raw_requests(requests: 1, project: project, file_path: file_path)
+
+            expect(response).to have_gitlab_http_status(200)
+          end
+        end
+
+        context 'when token is incorrect' do
+          it 'redirects to sign in page' do
+            request.headers['X-Gitlab-Static-Object-Token'] = 'foobar'
+            execute_raw_requests(requests: 1, project: project, file_path: file_path)
+
+            expect(response).to have_gitlab_http_status(302)
+            expect(response.location).to end_with('/users/sign_in')
+          end
+        end
+      end
+    end
   end
 
-  def execute_raw_requests(requests:, project:, file_path:)
+  def execute_raw_requests(requests:, project:, file_path:, **params)
     requests.times do
       get :show, params: {
         namespace_id: project.namespace,
         project_id: project,
         id: file_path
-      }
+      }.merge(params)
     end
   end
 end

spec/features/projects/blobs/blob_show_spec.rb

@@ -611,4 +611,50 @@ describe 'File blob', :js do
       expect(page).to have_selector '.gpg-status-box.invalid'
     end
   end
+
+  context 'when static objects external storage is enabled' do
+    before do
+      stub_application_setting(static_objects_external_storage_url: 'https://cdn.gitlab.com')
+    end
+
+    context 'private project' do
+      let_it_be(:project) { create(:project, :repository, :private) }
+      let_it_be(:user) { create(:user) }
+
+      before do
+        project.add_developer(user)
+
+        sign_in(user)
+        visit_blob('README.md')
+      end
+
+      it 'shows open raw and download buttons with external storage URL prepended and user token appended to their href' do
+        path = project_raw_path(project, 'master/README.md')
+        raw_uri = "https://cdn.gitlab.com#{path}?token=#{user.static_object_token}"
+        download_uri = "https://cdn.gitlab.com#{path}?inline=false&token=#{user.static_object_token}"
+
+        aggregate_failures do
+          expect(page).to have_link 'Open raw', href: raw_uri
+          expect(page).to have_link 'Download', href: download_uri
+        end
+      end
+    end
+
+    context 'public project' do
+      before do
+        visit_blob('README.md')
+      end
+
+      it 'shows open raw and download buttons with external storage URL prepended to their href' do
+        path = project_raw_path(project, 'master/README.md')
+        raw_uri = "https://cdn.gitlab.com#{path}"
+        download_uri = "https://cdn.gitlab.com#{path}?inline=false"
+
+        aggregate_failures do
+          expect(page).to have_link 'Open raw', href: raw_uri
+          expect(page).to have_link 'Download', href: download_uri
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/auth/user_auth_finders_spec.rb

@@ -116,9 +116,9 @@ describe Gitlab::Auth::UserAuthFinders do
   end
 
   describe '#find_user_from_static_object_token' do
-    context 'when request format is archive' do
+    shared_examples 'static object request' do
       before do
-        env['SCRIPT_NAME'] = 'project/-/archive/master.zip'
+        env['SCRIPT_NAME'] = path
       end
 
       context 'when token header param is present' do
@@ -126,7 +126,7 @@ describe Gitlab::Auth::UserAuthFinders do
           it 'returns the user' do
             request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token
 
-            expect(find_user_from_static_object_token(:archive)).to eq(user)
+            expect(find_user_from_static_object_token(format)).to eq(user)
           end
         end
 
@@ -134,7 +134,7 @@ describe Gitlab::Auth::UserAuthFinders do
           it 'returns the user' do
             request.headers['X-Gitlab-Static-Object-Token'] = 'foobar'
 
-            expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError)
+            expect { find_user_from_static_object_token(format) }.to raise_error(Gitlab::Auth::UnauthorizedError)
           end
         end
       end
@@ -144,7 +144,7 @@ describe Gitlab::Auth::UserAuthFinders do
           it 'returns the user' do
             set_param(:token, user.static_object_token)
 
-            expect(find_user_from_static_object_token(:archive)).to eq(user)
+            expect(find_user_from_static_object_token(format)).to eq(user)
           end
         end
 
@@ -152,13 +152,27 @@ describe Gitlab::Auth::UserAuthFinders do
           it 'returns the user' do
             set_param(:token, 'foobar')
 
-            expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError)
+            expect { find_user_from_static_object_token(format) }.to raise_error(Gitlab::Auth::UnauthorizedError)
           end
         end
       end
     end
 
-    context 'when request format is not archive' do
+    context 'when request format is archive' do
+      it_behaves_like 'static object request' do
+        let_it_be(:path) { 'project/-/archive/master.zip' }
+        let_it_be(:format) { :archive }
+      end
+    end
+
+    context 'when request format is blob' do
+      it_behaves_like 'static object request' do
+        let_it_be(:path) { 'project/raw/master/README.md' }
+        let_it_be(:format) { :blob }
+      end
+    end
+
+    context 'when request format is not archive nor blob' do
       before do
         env['script_name'] = 'url'
       end