Commit 054b4f2f authored by Douwe Maan's avatar Douwe Maan

Merge branch 'fj-48123-fix-gitlab-import' into 'master'

Fix gitlab import project load

Closes #48123

See merge request gitlab-org/gitlab-ce!20599
parents 480b5345 c52ab914
class Import::GitlabController < Import::BaseController class Import::GitlabController < Import::BaseController
MAX_PROJECT_PAGES = 15
PER_PAGE_PROJECTS = 100
before_action :verify_gitlab_import_enabled before_action :verify_gitlab_import_enabled
before_action :gitlab_auth, except: :callback before_action :gitlab_auth, except: :callback
...@@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController ...@@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController
end end
def status def status
@repos = client.projects @repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
@already_added_projects = find_already_added_projects('gitlab') @already_added_projects = find_already_added_projects('gitlab')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
......
---
title: Fix GitLab project imports not loading due to API timeouts
merge_request: 20599
author:
type: fixed
...@@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the ...@@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the
"avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png", "avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png",
"star_count":3812, "star_count":3812,
"forks_count":3561, "forks_count":3561,
"last_activity_at":"2018-01-02T11:40:26.570Z" "last_activity_at":"2018-01-02T11:40:26.570Z",
"namespace": {
"id": 72,
"name": "GitLab.org",
"path": "gitlab-org",
"kind": "group",
"full_path": "gitlab-org",
"parent_id": null
}
} }
... ...
......
...@@ -132,6 +132,7 @@ module API ...@@ -132,6 +132,7 @@ module API
expose :star_count, :forks_count expose :star_count, :forks_count
expose :last_activity_at expose :last_activity_at
expose :namespace, using: 'API::Entities::NamespaceBasic'
expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
def self.preload_relation(projects_relation, options = {}) def self.preload_relation(projects_relation, options = {})
...@@ -194,7 +195,6 @@ module API ...@@ -194,7 +195,6 @@ module API
expose :shared_runners_enabled expose :shared_runners_enabled
expose :lfs_enabled?, as: :lfs_enabled expose :lfs_enabled?, as: :lfs_enabled
expose :creator_id expose :creator_id
expose :namespace, using: 'API::Entities::NamespaceBasic'
expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
expose :import_status expose :import_status
expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
......
...@@ -32,15 +32,15 @@ module Gitlab ...@@ -32,15 +32,15 @@ module Gitlab
api.get("/api/v4/user").parsed api.get("/api/v4/user").parsed
end end
def issues(project_identifier) def issues(project_identifier, **kwargs)
lazy_page_iterator(PER_PAGE) do |page| lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
end end
end end
def issue_comments(project_identifier, issue_id) def issue_comments(project_identifier, issue_id, **kwargs)
lazy_page_iterator(PER_PAGE) do |page| lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
end end
end end
...@@ -48,23 +48,27 @@ module Gitlab ...@@ -48,23 +48,27 @@ module Gitlab
api.get("/api/v4/projects/#{id}").parsed api.get("/api/v4/projects/#{id}").parsed
end end
def projects def projects(**kwargs)
lazy_page_iterator(PER_PAGE) do |page| lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
end end
end end
private private
def lazy_page_iterator(per_page) def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
Enumerator.new do |y| Enumerator.new do |y|
page = 1 page = starting_page
page_limit = (starting_page - 1) + page_limit if page_limit
loop do loop do
items = yield(page) items = yield(page, per_page)
items.each do |item| items.each do |item|
y << item y << item
end end
break if items.empty? || items.size < per_page
break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
page += 1 page += 1
end end
......
...@@ -24,13 +24,24 @@ ...@@ -24,13 +24,24 @@
"avatar_url": { "type": ["string", "null"] }, "avatar_url": { "type": ["string", "null"] },
"star_count": { "type": "integer" }, "star_count": { "type": "integer" },
"forks_count": { "type": "integer" }, "forks_count": { "type": "integer" },
"last_activity_at": { "type": "date" } "last_activity_at": { "type": "date" },
"namespace": {
"type": "object",
"properties" : {
"id": { "type": "integer" },
"name": { "type": "string" },
"path": { "type": "string" },
"kind": { "type": "string" },
"full_path": { "type": "string" },
"parent_id": { "type": ["integer", "null"] }
}
}
}, },
"required": [ "required": [
"id", "name", "name_with_namespace", "description", "path", "id", "name", "name_with_namespace", "description", "path",
"path_with_namespace", "created_at", "default_branch", "tag_list", "path_with_namespace", "created_at", "default_branch", "tag_list",
"ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url", "ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
"star_count", "last_activity_at" "star_count", "last_activity_at", "namespace"
], ],
"additionalProperties": false "additionalProperties": false
} }
......
...@@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do ...@@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do
expect(key).to be_kind_of(Symbol) expect(key).to be_kind_of(Symbol)
end end
end end
it 'uses membership and simple flags' do
stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true')
expect_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
expect(client.projects.to_a).to eq []
end
shared_examples 'pagination params' do
before do
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
end
it 'allows page_limit param' do
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return(element_list)
expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original
client.send(method, *args, page_limit: 2, per_page: 1).to_a
end
it 'allows per_page param' do
expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original
client.send(method, *args, per_page: 2).to_a
end
it 'allows starting_page param' do
expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original
client.send(method, *args, starting_page: 3).to_a
end
end
describe '#projects' do
subject(:method) { :projects }
let(:args) { [] }
let(:element_list) { build_list(:project, 2) }
before do
stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true')
stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true')
stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true')
stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true')
end
it_behaves_like 'pagination params'
end
describe '#issues' do
subject(:method) { :issues }
let(:args) { [1] }
let(:element_list) { build_list(:issue, 2) }
before do
stub_request('/api/v4/projects/1/issues?page=1&per_page=1')
stub_request('/api/v4/projects/1/issues?page=2&per_page=1')
stub_request('/api/v4/projects/1/issues?page=1&per_page=2')
stub_request('/api/v4/projects/1/issues?page=3&per_page=100')
end
it_behaves_like 'pagination params'
end
describe '#issue_comments' do
subject(:method) { :issue_comments }
let(:args) { [1, 1] }
let(:element_list) { build_list(:note_on_issue, 2) }
before do
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1')
stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1')
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2')
stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100')
end
it_behaves_like 'pagination params'
end
def stub_request(path)
WebMock.stub_request(:get, "https://gitlab.com#{path}")
.to_return(status: 200)
end
end end
...@@ -20,7 +20,7 @@ describe API::Environments do ...@@ -20,7 +20,7 @@ describe API::Environments do
path path_with_namespace path path_with_namespace
star_count forks_count star_count forks_count
created_at last_activity_at created_at last_activity_at
avatar_url avatar_url namespace
) )
get api("/projects/#{project.id}/environments", user) get api("/projects/#{project.id}/environments", user)
......
...@@ -225,7 +225,7 @@ describe API::Projects do ...@@ -225,7 +225,7 @@ describe API::Projects do
path path_with_namespace path path_with_namespace
star_count forks_count star_count forks_count
created_at last_activity_at created_at last_activity_at
avatar_url avatar_url namespace
) )
get api('/projects?simple=true', user) get api('/projects?simple=true', user)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment