Commit c52ab914 authored by Francisco Javier López's avatar Francisco Javier López Committed by Douwe Maan

Fix gitlab import project load

parent 480b5345
class Import::GitlabController < Import::BaseController
MAX_PROJECT_PAGES = 15
PER_PAGE_PROJECTS = 100
before_action :verify_gitlab_import_enabled
before_action :gitlab_auth, except: :callback
......@@ -10,7 +13,7 @@ class Import::GitlabController < Import::BaseController
end
def status
@repos = client.projects
@repos = client.projects(starting_page: 1, page_limit: MAX_PROJECT_PAGES, per_page: PER_PAGE_PROJECTS)
@already_added_projects = find_already_added_projects('gitlab')
already_added_projects_names = @already_added_projects.pluck(:import_source)
......
---
title: Fix GitLab project imports not loading due to API timeouts
merge_request: 20599
author:
type: fixed
......@@ -573,7 +573,15 @@ If the project is a fork, and you provide a valid token to authenticate, the
"avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png",
"star_count":3812,
"forks_count":3561,
"last_activity_at":"2018-01-02T11:40:26.570Z"
"last_activity_at":"2018-01-02T11:40:26.570Z",
"namespace": {
"id": 72,
"name": "GitLab.org",
"path": "gitlab-org",
"kind": "group",
"full_path": "gitlab-org",
"parent_id": null
}
}
...
......
......@@ -132,6 +132,7 @@ module API
expose :star_count, :forks_count
expose :last_activity_at
expose :namespace, using: 'API::Entities::NamespaceBasic'
expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes
def self.preload_relation(projects_relation, options = {})
......@@ -194,7 +195,6 @@ module API
expose :shared_runners_enabled
expose :lfs_enabled?, as: :lfs_enabled
expose :creator_id
expose :namespace, using: 'API::Entities::NamespaceBasic'
expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
expose :import_status
expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
......
......@@ -32,15 +32,15 @@ module Gitlab
api.get("/api/v4/user").parsed
end
def issues(project_identifier)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
def issues(project_identifier, **kwargs)
lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed
end
end
def issue_comments(project_identifier, issue_id)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
def issue_comments(project_identifier, issue_id, **kwargs)
lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed
end
end
......@@ -48,23 +48,27 @@ module Gitlab
api.get("/api/v4/projects/#{id}").parsed
end
def projects
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
def projects(**kwargs)
lazy_page_iterator(**kwargs) do |page, per_page|
api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed
end
end
private
def lazy_page_iterator(per_page)
def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE)
Enumerator.new do |y|
page = 1
page = starting_page
page_limit = (starting_page - 1) + page_limit if page_limit
loop do
items = yield(page)
items = yield(page, per_page)
items.each do |item|
y << item
end
break if items.empty? || items.size < per_page
break if items.empty? || items.size < per_page || (page_limit && page >= page_limit)
page += 1
end
......
......@@ -24,13 +24,24 @@
"avatar_url": { "type": ["string", "null"] },
"star_count": { "type": "integer" },
"forks_count": { "type": "integer" },
"last_activity_at": { "type": "date" }
"last_activity_at": { "type": "date" },
"namespace": {
"type": "object",
"properties" : {
"id": { "type": "integer" },
"name": { "type": "string" },
"path": { "type": "string" },
"kind": { "type": "string" },
"full_path": { "type": "string" },
"parent_id": { "type": ["integer", "null"] }
}
}
},
"required": [
"id", "name", "name_with_namespace", "description", "path",
"path_with_namespace", "created_at", "default_branch", "tag_list",
"ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
"star_count", "last_activity_at"
"star_count", "last_activity_at", "namespace"
],
"additionalProperties": false
}
......
......@@ -15,4 +15,88 @@ describe Gitlab::GitlabImport::Client do
expect(key).to be_kind_of(Symbol)
end
end
it 'uses membership and simple flags' do
stub_request('/api/v4/projects?membership=true&page=1&per_page=100&simple=true')
expect_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
expect(client.projects.to_a).to eq []
end
shared_examples 'pagination params' do
before do
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return([])
end
it 'allows page_limit param' do
allow_any_instance_of(OAuth2::Response).to receive(:parsed).and_return(element_list)
expect(client).to receive(:lazy_page_iterator).with(hash_including(page_limit: 2)).and_call_original
client.send(method, *args, page_limit: 2, per_page: 1).to_a
end
it 'allows per_page param' do
expect(client).to receive(:lazy_page_iterator).with(hash_including(per_page: 2)).and_call_original
client.send(method, *args, per_page: 2).to_a
end
it 'allows starting_page param' do
expect(client).to receive(:lazy_page_iterator).with(hash_including(starting_page: 3)).and_call_original
client.send(method, *args, starting_page: 3).to_a
end
end
describe '#projects' do
subject(:method) { :projects }
let(:args) { [] }
let(:element_list) { build_list(:project, 2) }
before do
stub_request('/api/v4/projects?membership=true&page=1&per_page=1&simple=true')
stub_request('/api/v4/projects?membership=true&page=2&per_page=1&simple=true')
stub_request('/api/v4/projects?membership=true&page=1&per_page=2&simple=true')
stub_request('/api/v4/projects?membership=true&page=3&per_page=100&simple=true')
end
it_behaves_like 'pagination params'
end
describe '#issues' do
subject(:method) { :issues }
let(:args) { [1] }
let(:element_list) { build_list(:issue, 2) }
before do
stub_request('/api/v4/projects/1/issues?page=1&per_page=1')
stub_request('/api/v4/projects/1/issues?page=2&per_page=1')
stub_request('/api/v4/projects/1/issues?page=1&per_page=2')
stub_request('/api/v4/projects/1/issues?page=3&per_page=100')
end
it_behaves_like 'pagination params'
end
describe '#issue_comments' do
subject(:method) { :issue_comments }
let(:args) { [1, 1] }
let(:element_list) { build_list(:note_on_issue, 2) }
before do
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=1')
stub_request('/api/v4/projects/1/issues/1/notes?page=2&per_page=1')
stub_request('/api/v4/projects/1/issues/1/notes?page=1&per_page=2')
stub_request('/api/v4/projects/1/issues/1/notes?page=3&per_page=100')
end
it_behaves_like 'pagination params'
end
def stub_request(path)
WebMock.stub_request(:get, "https://gitlab.com#{path}")
.to_return(status: 200)
end
end
......@@ -20,7 +20,7 @@ describe API::Environments do
path path_with_namespace
star_count forks_count
created_at last_activity_at
avatar_url
avatar_url namespace
)
get api("/projects/#{project.id}/environments", user)
......
......@@ -225,7 +225,7 @@ describe API::Projects do
path path_with_namespace
star_count forks_count
created_at last_activity_at
avatar_url
avatar_url namespace
)
get api('/projects?simple=true', user)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment