Merge branch '11678-conan-job-tokens' into 'master'

Allow CI_JOB_TOKENs for Conan packages

Closes #11678

See merge request gitlab-org/gitlab!22184

changelogs/unreleased/11678-conan-job-tokens.yml

+---
+title: Allow CI_JOB_TOKENS for Conan package registry authentication
+merge_request: 22184
+author:
+type: added

doc/user/packages/conan_repository/index.md

@@ -223,7 +223,7 @@ There are two ways to remove a Conan package from the GitLab Package Registry.
   ```sh
   conan remove Hello/0.2@user/channel --remote=gitlab
   ```
-  
+
   You need to explicitly include the remote in this command, otherwise the package will only be removed from your
   local system cache.
 
@@ -263,3 +263,25 @@ The GitLab Conan repository supports the following Conan CLI commands:
 - `conan search`: Search the GitLab Package Registry for public packages, and private packages you have permission to view.
 - `conan info`: View the info on a given package from the GitLab Package Registry.
 - `conan remove`: Delete the package from the GitLab Package Registry.
+
+## Using GitLab CI with Conan packages
+
+To work with Conan commands within [GitLab CI](./../../../ci/README.md), you can use
+`CI_JOB_TOKEN` in place of the personal access token in your commands.
+
+It is easiest to provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each
+Conan command in your `.gitlab-ci.yml` file:
+
+```yml
+image: conanio/gcc7
+
+create_package:
+  stage: deploy
+  script:
+    - conan remote add gitlab https://gitlab.example.com/api/v4/packages/conan
+    - conan create . my-group+my-project/beta
+    - CONAN_LOGIN_USERNAME=ci_user CONAN_PASSWORD=${CI_JOB_TOKEN} conan upload Hello/0.1@root+ci-conan/beta1 --all --remote=gitlab
+```
+
+You can find additional Conan images to use as the base of your CI file
+in the [Conan docs](https://docs.conan.io/en/latest/howtos/run_conan_in_docker.html#available-docker-images).

ee/lib/api/conan_packages.rb

@@ -30,7 +30,7 @@ module API
       require_packages_enabled!
 
       # Personal access token will be extracted from Bearer or Basic authorization
-      # in the overridden find_personal_access_token helper
+      # in the overridden find_personal_access_token or find_user_from_job_token helpers
       authenticate!
     end
 
@@ -38,6 +38,7 @@ module API
       desc 'Ping the Conan API' do
         detail 'This feature was introduced in GitLab 12.2'
       end
+      route_setting :authentication, job_token_allowed: true
       get 'ping' do
         header 'X-Conan-Server-Capabilities', [].join(',')
       end
@@ -48,6 +49,7 @@ module API
       params do
         requires :q, type: String, desc: 'Search query'
       end
+      route_setting :authentication, job_token_allowed: true
       get 'conans/search' do
         service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute
         service.payload
@@ -59,14 +61,21 @@ module API
         desc 'Authenticate user against conan CLI' do
           detail 'This feature was introduced in GitLab 12.2'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'authenticate' do
-          token = ::Gitlab::ConanToken.from_personal_access_token(access_token)
+          token = if access_token
+                    ::Gitlab::ConanToken.from_personal_access_token(access_token)
+                  else
+                    ::Gitlab::ConanToken.from_job(find_job_from_token)
+                  end
+
           token.to_jwt
         end
 
         desc 'Check for valid user credentials per conan CLI' do
           detail 'This feature was introduced in GitLab 12.4'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'check_credentials' do
           authenticate!
           :ok
@@ -88,6 +97,7 @@ module API
         desc 'Package Snapshot' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'packages/:conan_package_reference' do
           authorize!(:read_package, project)
 
@@ -99,6 +109,7 @@ module API
         desc 'Recipe Snapshot' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get do
           authorize!(:read_package, project)
 
@@ -115,6 +126,7 @@ module API
         desc 'Package Digest' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'packages/:conan_package_reference/digest' do
           present_package_download_urls
         end
@@ -122,6 +134,7 @@ module API
         desc 'Recipe Digest' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'digest' do
           present_recipe_download_urls
         end
@@ -135,6 +148,7 @@ module API
         desc 'Package Download Urls' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'packages/:conan_package_reference/download_urls' do
           present_package_download_urls
         end
@@ -142,6 +156,7 @@ module API
         desc 'Recipe Download Urls' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         get 'download_urls' do
           present_recipe_download_urls
         end
@@ -159,6 +174,7 @@ module API
         params do
           requires :conan_package_reference, type: String, desc: 'Conan package ID'
         end
+        route_setting :authentication, job_token_allowed: true
         post 'packages/:conan_package_reference/upload_urls' do
           authorize!(:read_package, project)
 
@@ -171,6 +187,7 @@ module API
         desc 'Recipe Upload Urls' do
           detail 'This feature was introduced in GitLab 12.4'
         end
+        route_setting :authentication, job_token_allowed: true
         post 'upload_urls' do
           authorize!(:read_package, project)
 
@@ -183,6 +200,7 @@ module API
         desc 'Delete Package' do
           detail 'This feature was introduced in GitLab 12.5'
         end
+        route_setting :authentication, job_token_allowed: true
         delete do
           authorize!(:destroy_package, project)
 
@@ -211,6 +229,7 @@ module API
           desc 'Download recipe files' do
             detail 'This feature was introduced in GitLab 12.6'
           end
+          route_setting :authentication, job_token_allowed: true
           get do
             download_package_file(:recipe_file)
           end
@@ -221,6 +240,7 @@ module API
           params do
             use :workhorse_upload_params
           end
+          route_setting :authentication, job_token_allowed: true
           put do
             upload_package_file(:recipe_file)
           end
@@ -228,6 +248,7 @@ module API
           desc 'Workhorse authorize the conan recipe file' do
             detail 'This feature was introduced in GitLab 12.6'
           end
+          route_setting :authentication, job_token_allowed: true
           put 'authorize' do
             authorize_workhorse!(project)
           end
@@ -242,6 +263,7 @@ module API
           desc 'Download package files' do
             detail 'This feature was introduced in GitLab 12.5'
           end
+          route_setting :authentication, job_token_allowed: true
           get do
             download_package_file(:package_file)
           end
@@ -249,6 +271,7 @@ module API
           desc 'Workhorse authorize the conan package file' do
             detail 'This feature was introduced in GitLab 12.6'
           end
+          route_setting :authentication, job_token_allowed: true
           put 'authorize' do
             authorize_workhorse!(project)
           end
@@ -259,6 +282,7 @@ module API
           params do
             use :workhorse_upload_params
           end
+          route_setting :authentication, job_token_allowed: true
           put do
             upload_package_file(:package_file)
           end
@@ -386,7 +410,21 @@ module API
         personal_access_token = find_personal_access_token_from_conan_jwt ||
           find_personal_access_token_from_http_basic_auth
 
-        personal_access_token || unauthorized!
+        personal_access_token
+      end
+
+      def find_user_from_job_token
+        return unless route_authentication_setting[:job_token_allowed]
+
+        job = find_job_from_token
+
+        raise ::Gitlab::Auth::UnauthorizedError unless job
+
+        job.user
+      end
+
+      def find_job_from_token
+        find_job_from_conan_jwt || find_job_from_http_basic_auth
       end
 
       # We need to override this one because it
@@ -395,13 +433,31 @@ module API
       end
 
       def find_personal_access_token_from_conan_jwt
+        token = decode_oauth_token_from_jwt
+
+        return unless token
+
+        PersonalAccessToken.find_by_id_and_user_id(token.access_token_id, token.user_id)
+      end
+
+      def find_job_from_conan_jwt
+        token = decode_oauth_token_from_jwt
+
+        return unless token
+
+        ::Ci::Build.find_by_token(token.access_token_id.to_s)
+      end
+
+      def decode_oauth_token_from_jwt
         jwt = Doorkeeper::OAuth::Token.from_bearer_authorization(current_request)
+
         return unless jwt
 
         token = ::Gitlab::ConanToken.decode(jwt)
-        return unless token&.personal_access_token_id && token&.user_id
 
-        PersonalAccessToken.find_by_id_and_user_id(token.personal_access_token_id, token.user_id)
+        return unless token && token.access_token_id && token.user_id
+
+        token
       end
     end
   end

ee/lib/api/helpers/packages_manager_clients_helpers.rb

@@ -19,19 +19,35 @@ module API
       def find_personal_access_token_from_http_basic_auth
         return unless headers
 
-        encoded_credentials = headers['Authorization'].to_s.split('Basic ', 2).second
-        token = Base64.decode64(encoded_credentials || '').split(':', 2).second
+        token = decode_token
 
         return unless token
 
         PersonalAccessToken.find_by_token(token)
       end
 
+      def find_job_from_http_basic_auth
+        return unless headers
+
+        token = decode_token
+
+        return unless token
+
+        ::Ci::Build.find_by_token(token)
+      end
+
       def uploaded_package_file
         uploaded_file = UploadedFile.from_params(params, :file, ::Packages::PackageFileUploader.workhorse_local_upload_path)
         bad_request!('Missing package file!') unless uploaded_file
         uploaded_file
       end
+
+      private
+
+      def decode_token
+        encoded_credentials = headers['Authorization'].to_s.split('Basic ', 2).second
+        Base64.decode64(encoded_credentials || '').split(':', 2).second
+      end
     end
   end
 end

ee/lib/gitlab/conan_token.rb

 # frozen_string_literal: true
 
+# The Conan client uses a JWT for authenticating with remotes.
+# This class encodes and decodes a user's personal access token or
+# CI_JOB_TOKEN into a JWT that is used by the Conan client to
+# authenticate with GitLab
+
 module Gitlab
   class ConanToken
     HMAC_KEY = 'gitlab-conan-packages'.freeze
 
-    attr_reader :personal_access_token_id, :user_id
+    attr_reader :access_token_id, :user_id
 
     class << self
-      def from_personal_access_token(personal_access_token)
-        new(personal_access_token_id: personal_access_token.id, user_id: personal_access_token.user_id)
+      def from_personal_access_token(access_token)
+        new(access_token_id: access_token.id, user_id: access_token.user_id)
+      end
+
+      def from_job(job)
+        new(access_token_id: job.token, user_id: job.user.id)
       end
 
       def decode(jwt)
         payload = JSONWebToken::HMACToken.decode(jwt, secret).first
 
-        new(personal_access_token_id: payload['pat'], user_id: payload['u'])
-      rescue JWT::DecodeError
+        new(access_token_id: payload['access_token'], user_id: payload['user_id'])
+      rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature
+        # we return on expired and errored tokens because the Conan client
+        # will request a new token automatically.
       end
 
       def secret
@@ -27,8 +38,8 @@ module Gitlab
       end
     end
 
-    def initialize(personal_access_token_id:, user_id:)
-      @personal_access_token_id = personal_access_token_id
+    def initialize(access_token_id:, user_id:)
+      @access_token_id = access_token_id
       @user_id = user_id
     end
 
@@ -40,8 +51,8 @@ module Gitlab
 
     def hmac_token
       JSONWebToken::HMACToken.new(self.class.secret).tap do |token|
-        token['pat'] = personal_access_token_id
-        token['u'] = user_id
+        token['access_token'] = access_token_id
+        token['user_id'] = user_id
         token.expire_time = token.issued_at + 1.hour
       end
     end

ee/spec/lib/api/helpers/packages_manager_clients_helpers_spec.rb

@@ -42,6 +42,44 @@ describe API::Helpers::PackagesManagerClientsHelpers do
     end
   end
 
+  describe '#find_job_from_http_basic_auth' do
+    let_it_be(:user) { personal_access_token.user }
+
+    let(:job) { create(:ci_build, user: user) }
+    let(:password) { job.token }
+    let(:headers) { { Authorization: basic_http_auth(username, password) } }
+
+    subject { helper.find_job_from_http_basic_auth }
+
+    before do
+      allow(helper).to receive(:headers).and_return(headers&.with_indifferent_access)
+    end
+
+    context 'with a valid Authorization header' do
+      it { is_expected.to eq job }
+    end
+
+    context 'with an invalid Authorization header' do
+      where(:headers) do
+        [
+          [{ Authorization: 'Invalid' }],
+          [{}],
+          [nil]
+        ]
+      end
+
+      with_them do
+        it { is_expected.to be nil }
+      end
+    end
+
+    context 'with an unknown Authorization header' do
+      let(:password) { 'Unknown' }
+
+      it { is_expected.to be nil }
+    end
+  end
+
   describe '#uploaded_package_file' do
     let_it_be(:params) { {} }
 

ee/spec/lib/gitlab/conan_token_spec.rb

@@ -16,38 +16,58 @@ describe Gitlab::ConanToken do
     allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret)
   end
 
-  def build_jwt(personal_access_token_id:, user_id:)
+  def build_jwt(access_token_id:, user_id:, expire_time: nil)
     JSONWebToken::HMACToken.new(jwt_secret).tap do |jwt|
-      jwt['pat'] = personal_access_token_id
-      jwt['u'] = user_id || user_id
-      jwt.expire_time = jwt.issued_at + 1.hour
+      jwt['access_token'] = access_token_id
+      jwt['user_id'] = user_id || user_id
+      jwt.expire_time = expire_time || jwt.issued_at + 1.hour
     end
   end
 
   describe '.from_personal_access_token' do
-    it 'sets personal access token id and user id' do
-      personal_access_token = double(id: 123, user_id: 456)
+    it 'sets access token id and user id' do
+      access_token = double(id: 123, user_id: 456)
 
-      token = described_class.from_personal_access_token(personal_access_token)
+      token = described_class.from_personal_access_token(access_token)
 
-      expect(token.personal_access_token_id).to eq(123)
+      expect(token.access_token_id).to eq(123)
+      expect(token.user_id).to eq(456)
+    end
+  end
+
+  describe '.from_job' do
+    it 'sets access token id and user id' do
+      user = double(id: 456)
+      job = double(token: 123, user: user)
+
+      token = described_class.from_job(job)
+
+      expect(token.access_token_id).to eq(123)
       expect(token.user_id).to eq(456)
     end
   end
 
   describe '.decode' do
-    it 'sets personal access token id and user id' do
-      jwt = build_jwt(personal_access_token_id: 123, user_id: 456)
+    it 'sets access token id and user id' do
+      jwt = build_jwt(access_token_id: 123, user_id: 456)
 
       token = described_class.decode(jwt.encoded)
 
-      expect(token.personal_access_token_id).to eq(123)
+      expect(token.access_token_id).to eq(123)
       expect(token.user_id).to eq(456)
     end
 
     it 'returns nil for invalid JWT' do
       expect(described_class.decode('invalid-jwt')).to be_nil
     end
+
+    it 'returns nil for expired JWT' do
+      jwt = build_jwt(access_token_id: 123,
+                      user_id: 456,
+                      expire_time: Time.zone.now - 2.hours)
+
+      expect(described_class.decode(jwt.encoded)).to be_nil
+    end
   end
 
   describe '#to_jwt' do
@@ -55,9 +75,9 @@ describe Gitlab::ConanToken do
       allow(SecureRandom).to receive(:uuid).and_return('u-u-i-d')
 
       Timecop.freeze do
-        jwt = build_jwt(personal_access_token_id: 123, user_id: 456)
+        jwt = build_jwt(access_token_id: 123, user_id: 456)
 
-        token = described_class.new(personal_access_token_id: 123, user_id: 456)
+        token = described_class.new(access_token_id: 123, user_id: 456)
 
         expect(token.to_jwt).to eq(jwt.encoded)
       end

ee/spec/requests/api/conan_packages_spec.rb

@@ -12,6 +12,8 @@ describe API::ConanPackages do
 
   let(:base_secret) { SecureRandom.base64(64) }
   let(:auth_token) { personal_access_token.token }
+  let(:job) { create(:ci_build, user: user) }
+  let(:job_token) { job.token }
 
   let(:headers) do
     { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', auth_token) }
@@ -46,6 +48,14 @@ describe API::ConanPackages do
       expect(response.headers['X-Conan-Server-Capabilities']).to eq("")
     end
 
+    it 'responds with 200 OK when valid job token is provided' do
+      jwt = build_jwt_from_job(job)
+      get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded)
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response.headers['X-Conan-Server-Capabilities']).to eq("")
+    end
+
     it 'responds with 401 Unauthorized when invalid access token ID is provided' do
       jwt = build_jwt(double(id: 12345), user_id: personal_access_token.user_id)
       get api('/packages/conan/v1/ping'), headers: build_token_auth_header(jwt.encoded)
@@ -137,23 +147,43 @@ describe API::ConanPackages do
 
           payload = JSONWebToken::HMACToken.decode(
             response.body, jwt_secret).first
-          expect(payload['pat']).to eq(personal_access_token.id)
-          expect(payload['u']).to eq(personal_access_token.user_id)
+          expect(payload['access_token']).to eq(personal_access_token.id)
+          expect(payload['user_id']).to eq(personal_access_token.user_id)
 
           duration = payload['exp'] - payload['iat']
           expect(duration).to eq(1.hour)
         end
       end
     end
+
+    context 'with valid job token' do
+      let(:auth_token) { job_token }
+
+      it 'responds with 200' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+    end
   end
 
   describe 'GET /api/v4/packages/conan/v1/users/check_credentials' do
-    it 'responds with a 200 OK' do
+    it 'responds with a 200 OK with PAT' do
       get api('/packages/conan/v1/users/check_credentials'), headers: headers
 
       expect(response).to have_gitlab_http_status(:ok)
     end
 
+    context 'with job token' do
+      let(:auth_token) { job_token }
+
+      it 'responds with a 200 OK with job token' do
+        get api('/packages/conan/v1/users/check_credentials'), headers: headers
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+    end
+
     it 'responds with a 401 Unauthorized when an invalid token is used' do
       get api('/packages/conan/v1/users/check_credentials'), headers: build_token_auth_header('invalid-token')
 

ee/spec/support/helpers/ee/packages_manager_api_spec_helper.rb

@@ -16,8 +16,15 @@ module EE
 
     def build_jwt(personal_access_token, secret: jwt_secret, user_id: nil)
       JSONWebToken::HMACToken.new(secret).tap do |jwt|
-        jwt['pat'] = personal_access_token.id
-        jwt['u'] = user_id || personal_access_token.user_id
+        jwt['access_token'] = personal_access_token.id
+        jwt['user_id'] = user_id || personal_access_token.user_id
+      end
+    end
+
+    def build_jwt_from_job(job, secret: jwt_secret)
+      JSONWebToken::HMACToken.new(secret).tap do |jwt|
+        jwt['access_token'] = job.token
+        jwt['user_id'] = job.user.id
       end
     end