Merge branch 'fj-48123-fix-gitlab-import' into 'master'

Fix gitlab import project load

Closes #48123

See merge request gitlab-org/gitlab-ce!20599

app/controllers/import/gitlab_controller.rb

 class Import::GitlabController < Import::BaseController
+  MAX_PROJECT_PAGES = 15
+  PER_PAGE_PROJECTS = 100
+
   before_action :verify_gitlab_import_enabled
   before_action :gitlab_auth, except: :callback
 
@@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController
   end
 
   def status
-    @repos = client.projects
+    @repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
 
     @already_added_projects = find_already_added_projects('gitlab')
     already_added_projects_names = @already_added_projects.pluck(:import_source)

changelogs/unreleased/fj-48123-fix-gitlab-import.yml

+---
+title: Fix GitLab project imports not loading due to API timeouts
+merge_request: 20599
+author:
+type: fixed

doc/api/projects.md

@@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the
       "avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png",
       "star_count":3812,
       "forks_count":3561,
-      "last_activity_at":"2018-01-02T11:40:26.570Z"
+      "last_activity_at":"2018-01-02T11:40:26.570Z",
+      "namespace": {
+            "id": 72,
+            "name": "GitLab.org",
+            "path": "gitlab-org",
+            "kind": "group",
+            "full_path": "gitlab-org",
+            "parent_id": null
+      }
    }
 
    ...

lib/api/entities.rb

@@ -132,6 +132,7 @@ module API
       expose :star_count, :forks_count
       expose :last_activity_at
 
+      expose :namespace, using: 'API::Entities::NamespaceBasic'
       expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
 
       def self.preload_relation(projects_relation, options =  {})
@@ -194,7 +195,6 @@ module API
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace, using: 'API::Entities::NamespaceBasic'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
       expose :import_status
       expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }

lib/gitlab/gitlab_import/client.rb

@@ -32,15 +32,15 @@ module Gitlab
         api.get("/api/v4/user").parsed
       end
 
-      def issues(project_identifier)
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def issues(project_identifier, **kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
         end
       end
 
-      def issue_comments(project_identifier, issue_id)
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def issue_comments(project_identifier, issue_id, **kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
         end
       end
 
@@ -48,23 +48,27 @@ module Gitlab
         api.get("/api/v4/projects/#{id}").parsed
       end
 
-      def projects
-        lazy_page_iterator(PER_PAGE) do |page|
-          api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
+      def projects(**kwargs)
+        lazy_page_iterator(**kwargs) do |page, per_page|
+          api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
         end
       end
 
       private
 
-      def lazy_page_iterator(per_page)
+      def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
         Enumerator.new do |y|
-          page = 1
+          page = starting_page
+          page_limit = (starting_page - 1) + page_limit if page_limit
+
           loop do
-            items = yield(page)
+            items = yield(page, per_page)
+
             items.each do |item|
               y << item
             end
-            break if items.empty? || items.size < per_page
+
+            break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
 
             page += 1
           end

spec/fixtures/api/schemas/public_api/v4/projects.json

@@ -24,13 +24,24 @@
       "avatar_url": { "type": ["string", "null"] },
       "star_count": { "type": "integer" },
       "forks_count": { "type": "integer" },
-      "last_activity_at": { "type": "date" }
+      "last_activity_at": { "type": "date" },
+      "namespace": {
+        "type": "object",
+        "properties" : {
+          "id": { "type": "integer" },
+          "name": { "type": "string" },
+          "path": { "type": "string" },
+          "kind": { "type": "string" },
+          "full_path": { "type": "string" },
+          "parent_id": { "type": ["integer", "null"] }
+        }
+      }
     },
     "required": [
       "id", "name", "name_with_namespace", "description", "path",
       "path_with_namespace", "created_at", "default_branch", "tag_list",
       "ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
-      "star_count", "last_activity_at"
+      "star_count", "last_activity_at", "namespace"
     ],
     "additionalProperties": false
   }

spec/lib/gitlab/gitlab_import/client_spec.rb

@@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do
       expect(key).to be_kind_of(Symbol)
     end
   end
+
+  it 'uses membership and simple flags' do
+    stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true')
+
+    expect_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
+
+    expect(client.projects.to_a).to eq []
+  end
+
+  shared_examples 'pagination params' do
+    before do
+      allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
+    end
+
+    it 'allows page_limit param' do
+      allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return(element_list)
+
+      expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original
+
+      client.send(method, *args, page_limit: 2, per_page: 1).to_a
+    end
+
+    it 'allows per_page param' do
+      expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original
+
+      client.send(method, *args, per_page: 2).to_a
+    end
+
+    it 'allows starting_page param' do
+      expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original
+
+      client.send(method, *args, starting_page: 3).to_a
+    end
+  end
+
+  describe '#projects' do
+    subject(:method) { :projects }
+    let(:args) { [] }
+    let(:element_list) { build_list(:project, 2) }
+
+    before do
+      stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true')
+      stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true')
+      stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true')
+      stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true')
+    end
+
+    it_behaves_like 'pagination params'
+  end
+
+  describe '#issues' do
+    subject(:method) { :issues }
+    let(:args) { [1] }
+    let(:element_list) { build_list(:issue, 2) }
+
+    before do
+      stub_request('/api/v4/projects/1/issues?page=1&per_page=1')
+      stub_request('/api/v4/projects/1/issues?page=2&per_page=1')
+      stub_request('/api/v4/projects/1/issues?page=1&per_page=2')
+      stub_request('/api/v4/projects/1/issues?page=3&per_page=100')
+    end
+
+    it_behaves_like 'pagination params'
+  end
+
+  describe '#issue_comments' do
+    subject(:method) { :issue_comments }
+    let(:args) { [1, 1] }
+    let(:element_list) { build_list(:note_on_issue, 2) }
+
+    before do
+      stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1')
+      stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1')
+      stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2')
+      stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100')
+    end
+
+    it_behaves_like 'pagination params'
+  end
+
+  def stub_request(path)
+    WebMock.stub_request(:get, "https://gitlab.com#{path}")
+      .to_return(status: 200)
+  end
 end

spec/requests/api/environments_spec.rb

@@ -20,7 +20,7 @@ describe API::Environments do
           path path_with_namespace
           star_count forks_count
           created_at last_activity_at
-          avatar_url
+          avatar_url namespace
         )
 
         get api("/projects/#{project.id}/environments", user)

spec/requests/api/projects_spec.rb

@@ -225,7 +225,7 @@ describe API::Projects do
             path path_with_namespace
             star_count forks_count
             created_at last_activity_at
-            avatar_url
+            avatar_url namespace
           )
 
           get api('/projects?simple=true', user)