Merge branch 'georgekoltsov/remove-legacy-github-client-v2' into 'master'

Replace legacy github client in GitHub Controller

See merge request gitlab-org/gitlab!38915

app/controllers/import/gitea_controller.rb

@@ -54,6 +54,16 @@ class Import::GiteaController < Import::GithubController
     end
   end
 
+  override :client_repos
+  def client_repos
+    @client_repos ||= filtered(client.repos)
+  end
+
+  override :client
+  def client
+    @client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options)
+  end
+
   override :client_options
   def client_options
     { host: provider_url, api_version: 'v1' }

app/controllers/import/github_controller.rb

@@ -10,6 +10,9 @@ class Import::GithubController < Import::BaseController
   before_action :provider_auth, only: [:status, :realtime_changes, :create]
   before_action :expire_etag_cache, only: [:status, :create]
 
+  OAuthConfigMissingError = Class.new(StandardError)
+
+  rescue_from OAuthConfigMissingError, with: :missing_oauth_config
   rescue_from Octokit::Unauthorized, with: :provider_unauthorized
   rescue_from Octokit::TooManyRequests, with: :provider_rate_limit
 
@@ -22,7 +25,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def callback
-    session[access_token_key] = client.get_token(params[:code])
+    session[access_token_key] = get_token(params[:code])
     redirect_to status_import_url
   end
 
@@ -77,9 +80,7 @@ class Import::GithubController < Import::BaseController
   override :provider_url
   def provider_url
     strong_memoize(:provider_url) do
-      provider = Gitlab::Auth::OAuth::Provider.config_for('github')
-
-      provider&.dig('url').presence || 'https://github.com'
+      oauth_config&.dig('url').presence || 'https://github.com'
     end
   end
 
@@ -104,11 +105,66 @@ class Import::GithubController < Import::BaseController
   end
 
   def client
-    @client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options)
+    @client ||= if Feature.enabled?(:remove_legacy_github_client)
+                  Gitlab::GithubImport::Client.new(session[access_token_key])
+                else
+                  Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options)
+                end
   end
 
   def client_repos
-    @client_repos ||= filtered(client.repos)
+    @client_repos ||= if Feature.enabled?(:remove_legacy_github_client)
+                        filtered(concatenated_repos)
+                      else
+                        filtered(client.repos)
+                      end
+  end
+
+  def concatenated_repos
+    return [] unless client.respond_to?(:each_page)
+
+    client.each_page(:repos).flat_map(&:objects)
+  end
+
+  def oauth_client
+    raise OAuthConfigMissingError unless oauth_config
+
+    @oauth_client ||= ::OAuth2::Client.new(
+      oauth_config.app_id,
+      oauth_config.app_secret,
+      oauth_options.merge(ssl: { verify: oauth_config['verify_ssl'] })
+    )
+  end
+
+  def oauth_config
+    @oauth_config ||= Gitlab::Auth::OAuth::Provider.config_for('github')
+  end
+
+  def oauth_options
+    if oauth_config
+      oauth_config.dig('args', 'client_options').deep_symbolize_keys
+    else
+      OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys
+    end
+  end
+
+  def authorize_url
+    if Feature.enabled?(:remove_legacy_github_client)
+      oauth_client.auth_code.authorize_url(
+        redirect_uri: callback_import_url,
+        scope: 'repo, user, user:email'
+      )
+    else
+      client.authorize_url(callback_import_url)
+    end
+  end
+
+  def get_token(code)
+    if Feature.enabled?(:remove_legacy_github_client)
+      oauth_client.auth_code.get_token(code).token
+    else
+      client.get_token(code)
+    end
   end
 
   def verify_import_enabled
@@ -116,7 +172,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def go_to_provider_for_permissions
-    redirect_to client.authorize_url(callback_import_url)
+    redirect_to authorize_url
   end
 
   def import_enabled?
@@ -152,6 +208,12 @@ class Import::GithubController < Import::BaseController
       alert: _("GitHub API rate limit exceeded. Try again after %{reset_time}") % { reset_time: reset_time }
   end
 
+  def missing_oauth_config
+    session[access_token_key] = nil
+    redirect_to new_import_url,
+      alert: _('Missing OAuth configuration for GitHub.')
+  end
+
   def access_token_key
     :"#{provider_name}_access_token"
   end

app/services/import/github_service.rb

@@ -33,7 +33,7 @@ module Import
     end
 
     def repo
-      @repo ||= client.repo(params[:repo_id].to_i)
+      @repo ||= client.repository(params[:repo_id].to_i)
     end
 
     def project_name

ee/spec/features/projects/new_project_spec.rb

@@ -53,6 +53,7 @@ RSpec.describe 'New project' do
     context 'when licensed' do
       before do
         stub_licensed_features(ci_cd_projects: true)
+        stub_feature_flags(remove_legacy_github_client: false)
       end
 
       it 'shows CI/CD tab and pane' do
@@ -122,7 +123,7 @@ RSpec.describe 'New project' do
         wait_for_requests
 
         # Mock the POST `/import/github`
-        allow_any_instance_of(Gitlab::LegacyGithubImport::Client).to receive(:repo).and_return(repo)
+        allow_any_instance_of(Gitlab::LegacyGithubImport::Client).to receive(:repository).and_return(repo)
         project = create(:project, name: 'some-github-repo', creator: user, import_type: 'github')
         create(:import_state, :finished, import_url: repo.clone_url, project: project)
         allow_any_instance_of(CiCd::SetupProject).to receive(:setup_external_service)

lib/api/import_github.rb

@@ -10,7 +10,11 @@ module API
 
     helpers do
       def client
-        @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
+        @client ||= if Feature.enabled?(:remove_legacy_github_client)
+                      Gitlab::GithubImport::Client.new(params[:personal_access_token])
+                    else
+                      Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
+                    end
       end
 
       def access_params

locale/gitlab.pot

@@ -15642,6 +15642,9 @@ msgstr ""
 msgid "Mirroring will only be available if the feature is included in the plan of the selected group or user."
 msgstr ""
 
+msgid "Missing OAuth configuration for GitHub."
+msgstr ""
+
 msgid "Missing commit signatures endpoint!"
 msgstr ""
 

spec/controllers/import/gitea_controller_spec.rb

@@ -34,6 +34,14 @@ RSpec.describe Import::GiteaController do
         assign_host_url
       end
 
+      it "requests provider repos list" do
+        expect(stub_client(repos: [], orgs: [])).to receive(:repos)
+
+        get :status
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+
       context 'when host url is local or not http' do
         %w[https://localhost:3000 http://192.168.0.1 ftp://testing].each do |url|
           let(:host_url) { url }

spec/controllers/import/github_controller_spec.rb

@@ -15,10 +15,7 @@ RSpec.describe Import::GithubController do
     it "redirects to GitHub for an access token if logged in with GitHub" do
       allow(controller).to receive(:logged_in_with_provider?).and_return(true)
       expect(controller).to receive(:go_to_provider_for_permissions).and_call_original
-      allow_any_instance_of(Gitlab::LegacyGithubImport::Client)
-        .to receive(:authorize_url)
-        .with(users_import_github_callback_url)
-        .and_call_original
+      allow(controller).to receive(:authorize_url).and_call_original
 
       get :new
 
@@ -46,13 +43,15 @@ RSpec.describe Import::GithubController do
   end
 
   describe "GET callback" do
+    before do
+      allow(controller).to receive(:get_token).and_return(token)
+      allow(controller).to receive(:oauth_options).and_return({})
+
+      stub_omniauth_provider('github')
+    end
+
     it "updates access token" do
       token = "asdasd12345"
-      allow_any_instance_of(Gitlab::LegacyGithubImport::Client)
-        .to receive(:get_token).and_return(token)
-      allow_any_instance_of(Gitlab::LegacyGithubImport::Client)
-        .to receive(:github_options).and_return({})
-      stub_omniauth_provider('github')
 
       get :callback
 
@@ -66,7 +65,86 @@ RSpec.describe Import::GithubController do
   end
 
   describe "GET status" do
+    context 'when using OAuth' do
+      before do
+        allow(controller).to receive(:logged_in_with_provider?).and_return(true)
+      end
+
+      context 'when OAuth config is missing' do
+        let(:new_import_url) { public_send("new_import_#{provider}_url") }
+
+        before do
+          allow(controller).to receive(:oauth_config).and_return(nil)
+        end
+
+        it 'returns missing config error' do
+          expect(controller).to receive(:go_to_provider_for_permissions).and_call_original
+
+          get :status
+
+          expect(session[:"#{provider}_access_token"]).to be_nil
+          expect(controller).to redirect_to(new_import_url)
+          expect(flash[:alert]).to eq('Missing OAuth configuration for GitHub.')
+        end
+      end
+    end
+
+    context 'when feature remove_legacy_github_client is disabled' do
+      before do
+        stub_feature_flags(remove_legacy_github_client: false)
+        session[:"#{provider}_access_token"] = 'asdasd12345'
+      end
+
+      it_behaves_like 'a GitHub-ish import controller: GET status'
+
+      it 'uses Gitlab::LegacyGitHubImport::Client' do
+        expect(controller.send(:client)).to be_instance_of(Gitlab::LegacyGithubImport::Client)
+      end
+
+      it 'fetches repos using legacy client' do
+        expect_next_instance_of(Gitlab::LegacyGithubImport::Client) do |client|
+          expect(client).to receive(:repos)
+        end
+
+        get :status
+      end
+    end
+
+    context 'when feature remove_legacy_github_client is enabled' do
+      before do
+        stub_feature_flags(remove_legacy_github_client: true)
+        session[:"#{provider}_access_token"] = 'asdasd12345'
+      end
+
       it_behaves_like 'a GitHub-ish import controller: GET status'
+
+      it 'uses Gitlab::GithubImport::Client' do
+        expect(controller.send(:client)).to be_instance_of(Gitlab::GithubImport::Client)
+      end
+
+      it 'fetches repos using latest github client' do
+        expect_next_instance_of(Gitlab::GithubImport::Client) do |client|
+          expect(client).to receive(:each_page).with(:repos).and_return([].to_enum)
+        end
+
+        get :status
+      end
+
+      it 'concatenates list of repos from multiple pages' do
+        repo_1 = OpenStruct.new(login: 'emacs', full_name: 'asd/emacs', name: 'emacs', owner: { login: 'owner' })
+        repo_2 = OpenStruct.new(login: 'vim', full_name: 'asd/vim', name: 'vim', owner: { login: 'owner' })
+        repos = [OpenStruct.new(objects: [repo_1]), OpenStruct.new(objects: [repo_2])].to_enum
+
+        allow(stub_client).to receive(:each_page).and_return(repos)
+
+        get :status, format: :json
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response.dig('provider_repos').count).to eq(2)
+        expect(json_response.dig('provider_repos', 0, 'id')).to eq(repo_1.id)
+        expect(json_response.dig('provider_repos', 1, 'id')).to eq(repo_2.id)
+      end
+    end
   end
 
   describe "POST create" do

spec/requests/api/import_github_spec.rb

@@ -22,7 +22,7 @@ RSpec.describe API::ImportGithub do
 
     before do
       Grape::Endpoint.before_each do |endpoint|
-        allow(endpoint).to receive(:client).and_return(double('client', user: provider_user, repo: provider_repo).as_null_object)
+        allow(endpoint).to receive(:client).and_return(double('client', user: provider_user, repository: provider_repo).as_null_object)
       end
     end
 

spec/services/import/github_service_spec.rb

@@ -6,7 +6,6 @@ RSpec.describe Import::GithubService do
   let_it_be(:user) { create(:user) }
   let_it_be(:token) { 'complex-token' }
   let_it_be(:access_params) { { github_access_token: 'github-complex-token' } }
-  let_it_be(:client) { Gitlab::LegacyGithubImport::Client.new(token) }
   let_it_be(:params) { { repo_id: 123, new_name: 'new_repo', target_namespace: 'root' } }
 
   let(:subject) { described_class.new(client, user, params) }
@@ -15,11 +14,14 @@ RSpec.describe Import::GithubService do
     allow(subject).to receive(:authorized?).and_return(true)
   end
 
+  shared_examples 'handles errors' do |klass|
+    let(:client) { klass.new(token) }
+
     context 'do not raise an exception on input error' do
       let(:exception) { Octokit::ClientError.new(status: 404, body: 'Not Found') }
 
       before do
-      expect(client).to receive(:repo).and_raise(exception)
+        expect(client).to receive(:repository).and_raise(exception)
       end
 
       it 'logs the original error' do
@@ -46,10 +48,27 @@ RSpec.describe Import::GithubService do
     it 'raises an exception for unknown error causes' do
       exception = StandardError.new('Not Implemented')
 
-    expect(client).to receive(:repo).and_raise(exception)
+      expect(client).to receive(:repository).and_raise(exception)
 
       expect(Gitlab::Import::Logger).not_to receive(:error)
 
       expect { subject.execute(access_params, :github) }.to raise_error(exception)
     end
+  end
+
+  context 'when remove_legacy_github_client feature flag is enabled' do
+    before do
+      stub_feature_flags(remove_legacy_github_client: true)
+    end
+
+    include_examples 'handles errors', Gitlab::GithubImport::Client
+  end
+
+  context 'when remove_legacy_github_client feature flag is enabled' do
+    before do
+      stub_feature_flags(remove_legacy_github_client: false)
+    end
+
+    include_examples 'handles errors', Gitlab::LegacyGithubImport::Client
+  end
 end

spec/support/shared_examples/controllers/githubish_import_controller_shared_examples.rb

@@ -72,7 +72,7 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
     project = create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'example/repo')
     group = create(:group)
     group.add_owner(user)
-    stub_client(repos: [repo, org_repo], orgs: [org], org_repos: [org_repo])
+    stub_client(repos: [repo, org_repo], orgs: [org], org_repos: [org_repo], each_page: [OpenStruct.new(objects: [repo, org_repo])].to_enum)
 
     get :status, format: :json
 
@@ -85,7 +85,7 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
 
   it "does not show already added project" do
     project = create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'asd/vim')
-    stub_client(repos: [repo], orgs: [])
+    stub_client(repos: [repo], orgs: [], each_page: [OpenStruct.new(objects: [repo])].to_enum)
 
     get :status, format: :json
 
@@ -94,7 +94,8 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
   end
 
   it "touches the etag cache store" do
-    expect(stub_client(repos: [], orgs: [])).to receive(:repos)
+    stub_client(repos: [], orgs: [], each_page: [])
+
     expect_next_instance_of(Gitlab::EtagCaching::Store) do |store|
       expect(store).to receive(:touch) { "realtime_changes_import_#{provider}_path" }
     end
@@ -102,17 +103,11 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
     get :status, format: :json
   end
 
-  it "requests provider repos list" do
-    expect(stub_client(repos: [], orgs: [])).to receive(:repos)
-
-    get :status
-
-    expect(response).to have_gitlab_http_status(:ok)
-  end
-
   it "handles an invalid access token" do
-    allow_any_instance_of(Gitlab::LegacyGithubImport::Client)
-      .to receive(:repos).and_raise(Octokit::Unauthorized)
+    client = stub_client(repos: [], orgs: [], each_page: [])
+
+    allow(client).to receive(:repos).and_raise(Octokit::Unauthorized)
+    allow(client).to receive(:each_page).and_raise(Octokit::Unauthorized)
 
     get :status
 
@@ -122,7 +117,7 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
   end
 
   it "does not produce N+1 database queries" do
-    stub_client(repos: [repo], orgs: [])
+    stub_client(repos: [repo], orgs: [], each_page: [].to_enum)
     group_a = create(:group)
     group_a.add_owner(user)
     create(:project, :import_started, import_type: provider, namespace: user.namespace)
@@ -144,10 +139,12 @@ RSpec.shared_examples 'a GitHub-ish import controller: GET status' do
     let(:repo_2) { OpenStruct.new(login: 'emacs', full_name: 'asd/emacs', name: 'emacs', owner: { login: 'owner' }) }
     let(:project) { create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'example/repo') }
     let(:group) { create(:group) }
+    let(:repos) { [repo, repo_2, org_repo] }
 
     before do
       group.add_owner(user)
-      stub_client(repos: [repo, repo_2, org_repo], orgs: [org], org_repos: [org_repo])
+      client = stub_client(repos: repos, orgs: [org], org_repos: [org_repo])
+      allow(client).to receive(:each_page).and_return([OpenStruct.new(objects: repos)].to_enum)
     end
 
     it 'filters list of repositories by name' do
@@ -187,7 +184,7 @@ RSpec.shared_examples 'a GitHub-ish import controller: POST create' do
   end
 
   before do
-    stub_client(user: provider_user, repo: provider_repo)
+    stub_client(user: provider_user, repo: provider_repo, repository: provider_repo)
     assign_session_token(provider)
   end