Commit 5fcd7a87 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch 'sh-paginate-bitbucket-server-imports' into 'master'

Paginate Bitbucket Server importer projects

See merge request gitlab-org/gitlab-ce!22825
parents 829e4525 5b6d5301
...@@ -54,14 +54,14 @@ class Import::BitbucketServerController < Import::BaseController ...@@ -54,14 +54,14 @@ class Import::BitbucketServerController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def status def status
repos = bitbucket_client.repos @collection = bitbucket_client.repos(page_offset: page_offset, limit: limit_per_page)
@repos, @incompatible_repos = @collection.partition { |repo| repo.valid? }
@repos, @incompatible_repos = repos.partition { |repo| repo.valid? } # Use the import URL to filter beyond what BaseService#find_already_added_projects
@already_added_projects = filter_added_projects('bitbucket_server', @repos.map(&:browse_url))
@already_added_projects = find_already_added_projects('bitbucket_server')
already_added_projects_names = @already_added_projects.pluck(:import_source) already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.browse_url) } @repos.reject! { |repo| already_added_projects_names.include?(repo.browse_url) }
rescue BitbucketServer::Connection::ConnectionError, BitbucketServer::Client::ServerError => e rescue BitbucketServer::Connection::ConnectionError, BitbucketServer::Client::ServerError => e
flash[:alert] = "Unable to connect to server: #{e}" flash[:alert] = "Unable to connect to server: #{e}"
clear_session_data clear_session_data
...@@ -75,6 +75,12 @@ class Import::BitbucketServerController < Import::BaseController ...@@ -75,6 +75,12 @@ class Import::BitbucketServerController < Import::BaseController
private private
# rubocop: disable CodeReuse/ActiveRecord
def filter_added_projects(import_type, import_sources)
current_user.created_projects.where(import_type: import_type, import_source: import_sources).includes(:import_state)
end
# rubocop: enable CodeReuse/ActiveRecord
def bitbucket_client def bitbucket_client
@bitbucket_client ||= BitbucketServer::Client.new(credentials) @bitbucket_client ||= BitbucketServer::Client.new(credentials)
end end
...@@ -130,4 +136,12 @@ class Import::BitbucketServerController < Import::BaseController ...@@ -130,4 +136,12 @@ class Import::BitbucketServerController < Import::BaseController
password: session[personal_access_token_key] password: session[personal_access_token_key]
} }
end end
def page_offset
[0, params[:page].to_i].max
end
def limit_per_page
BitbucketServer::Paginator::PAGE_LENGTH
end
end end
...@@ -84,4 +84,6 @@ ...@@ -84,4 +84,6 @@
= link_to 'import flow', status_import_bitbucket_server_path = link_to 'import flow', status_import_bitbucket_server_path
again. again.
= paginate_without_count(@collection)
.js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_server_path}", import_path: "#{import_bitbucket_server_path}" } } .js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_server_path}", import_path: "#{import_bitbucket_server_path}" } }
---
title: Paginate Bitbucket Server importer projects
merge_request: 22825
author:
type: changed
...@@ -35,9 +35,9 @@ module BitbucketServer ...@@ -35,9 +35,9 @@ module BitbucketServer
BitbucketServer::Representation::Repo.new(parsed_response) BitbucketServer::Representation::Repo.new(parsed_response)
end end
def repos def repos(page_offset: 0, limit: nil)
path = "/repos" path = "/repos"
get_collection(path, :repo) get_collection(path, :repo, page_offset: page_offset, limit: limit)
end end
def create_branch(project_key, repo, branch_name, sha) def create_branch(project_key, repo, branch_name, sha)
...@@ -61,8 +61,8 @@ module BitbucketServer ...@@ -61,8 +61,8 @@ module BitbucketServer
private private
def get_collection(path, type) def get_collection(path, type, page_offset: 0, limit: nil)
paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type) paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit)
BitbucketServer::Collection.new(paginator) BitbucketServer::Collection.new(paginator)
rescue *SERVER_ERRORS => e rescue *SERVER_ERRORS => e
raise ServerError, e raise ServerError, e
......
...@@ -2,7 +2,13 @@ ...@@ -2,7 +2,13 @@
module BitbucketServer module BitbucketServer
class Collection < Enumerator class Collection < Enumerator
attr_reader :paginator
delegate :page_offset, :has_next_page?, to: :paginator
def initialize(paginator) def initialize(paginator)
@paginator = paginator
super() do |yielder| super() do |yielder|
loop do loop do
paginator.items.each { |item| yielder << item } paginator.items.each { |item| yielder << item }
...@@ -12,6 +18,24 @@ module BitbucketServer ...@@ -12,6 +18,24 @@ module BitbucketServer
lazy lazy
end end
def current_page
return 1 if page_offset <= 1
[1, page_offset].max
end
def prev_page
return nil unless current_page > 1
current_page - 1
end
def next_page
return nil unless has_next_page?
current_page + 1
end
def method_missing(method, *args) def method_missing(method, *args)
return super unless self.respond_to?(method) return super unless self.respond_to?(method)
......
...@@ -4,34 +4,49 @@ module BitbucketServer ...@@ -4,34 +4,49 @@ module BitbucketServer
class Paginator class Paginator
PAGE_LENGTH = 25 PAGE_LENGTH = 25
def initialize(connection, url, type) attr_reader :page_offset
def initialize(connection, url, type, page_offset: 0, limit: nil)
@connection = connection @connection = connection
@type = type @type = type
@url = url @url = url
@page = nil @page = nil
@page_offset = page_offset
@limit = limit || PAGE_LENGTH
@total = 0
end end
def items def items
raise StopIteration unless has_next_page? raise StopIteration unless has_next_page?
raise StopIteration if over_limit?
@page = fetch_next_page @page = fetch_next_page
@total += @page.items.count
@page.items @page.items
end end
def has_next_page?
page.nil? || page.next?
end
private private
attr_reader :connection, :page, :url, :type attr_reader :connection, :page, :url, :type, :limit
def has_next_page? def over_limit?
page.nil? || page.next? @limit.positive? && @total >= @limit
end end
def next_offset def next_offset
page.nil? ? 0 : page.next page.nil? ? starting_offset : page.next
end
def starting_offset
[0, page_offset - 1].max * limit
end end
def fetch_next_page def fetch_next_page
parsed_response = connection.get(@url, start: next_offset, limit: PAGE_LENGTH) parsed_response = connection.get(@url, start: next_offset, limit: @limit)
Page.new(parsed_response, type) Page.new(parsed_response, type)
end end
end end
......
...@@ -121,12 +121,19 @@ describe Import::BitbucketServerController do ...@@ -121,12 +121,19 @@ describe Import::BitbucketServerController do
@repo = double(slug: 'vim', project_key: 'asd', full_name: 'asd/vim', "valid?" => true, project_name: 'asd', browse_url: 'http://test', name: 'vim') @repo = double(slug: 'vim', project_key: 'asd', full_name: 'asd/vim', "valid?" => true, project_name: 'asd', browse_url: 'http://test', name: 'vim')
@invalid_repo = double(slug: 'invalid', project_key: 'foobar', full_name: 'asd/foobar', "valid?" => false, browse_url: 'http://bad-repo') @invalid_repo = double(slug: 'invalid', project_key: 'foobar', full_name: 'asd/foobar', "valid?" => false, browse_url: 'http://bad-repo')
@created_repo = double(slug: 'created', project_key: 'existing', full_name: 'group/created', "valid?" => true, browse_url: 'http://existing')
assign_session_tokens assign_session_tokens
end end
it 'assigns repository categories' do it 'assigns repository categories' do
created_project = create(:project, import_type: 'bitbucket_server', creator_id: user.id, import_source: 'foo/bar', import_status: 'finished') created_project = create(:project, import_type: 'bitbucket_server', creator_id: user.id, import_status: 'finished', import_source: @created_repo.browse_url)
expect(client).to receive(:repos).and_return([@repo, @invalid_repo]) repos = instance_double(BitbucketServer::Collection)
expect(repos).to receive(:partition).and_return([[@repo, @created_repo], [@invalid_repo]])
expect(repos).to receive(:current_page).and_return(1)
expect(repos).to receive(:next_page).and_return(2)
expect(repos).to receive(:prev_page).and_return(nil)
expect(client).to receive(:repos).and_return(repos)
get :status get :status
......
...@@ -13,7 +13,7 @@ describe BitbucketServer::Client do ...@@ -13,7 +13,7 @@ describe BitbucketServer::Client do
let(:path) { "/projects/#{project}/repos/#{repo_slug}/pull-requests?state=ALL" } let(:path) { "/projects/#{project}/repos/#{repo_slug}/pull-requests?state=ALL" }
it 'requests a collection' do it 'requests a collection' do
expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :pull_request) expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :pull_request, page_offset: 0, limit: nil)
subject.pull_requests(project, repo_slug) subject.pull_requests(project, repo_slug)
end end
...@@ -29,7 +29,7 @@ describe BitbucketServer::Client do ...@@ -29,7 +29,7 @@ describe BitbucketServer::Client do
let(:path) { "/projects/#{project}/repos/#{repo_slug}/pull-requests/1/activities" } let(:path) { "/projects/#{project}/repos/#{repo_slug}/pull-requests/1/activities" }
it 'requests a collection' do it 'requests a collection' do
expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :activity) expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :activity, page_offset: 0, limit: nil)
subject.activities(project, repo_slug, 1) subject.activities(project, repo_slug, 1)
end end
...@@ -52,10 +52,16 @@ describe BitbucketServer::Client do ...@@ -52,10 +52,16 @@ describe BitbucketServer::Client do
let(:path) { "/repos" } let(:path) { "/repos" }
it 'requests a collection' do it 'requests a collection' do
expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :repo) expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :repo, page_offset: 0, limit: nil)
subject.repos subject.repos
end end
it 'requests a collection with an offset and limit' do
expect(BitbucketServer::Paginator).to receive(:new).with(anything, path, :repo, page_offset: 10, limit: 25)
subject.repos(page_offset: 10, limit: 25)
end
end end
describe '#create_branch' do describe '#create_branch' do
......
# frozen_string_literal: true
require 'spec_helper'
describe BitbucketServer::Collection do
let(:connection) { instance_double(BitbucketServer::Connection) }
let(:page) { 1 }
let(:paginator) { BitbucketServer::Paginator.new(connection, 'http://more-data', :pull_request, page_offset: page) }
subject { described_class.new(paginator) }
describe '#current_page' do
it 'returns 1' do
expect(subject.current_page).to eq(1)
end
end
describe '#prev_page' do
it 'returns nil' do
expect(subject.prev_page).to be_nil
end
end
describe '#next_page' do
it 'returns 2' do
expect(subject.next_page).to eq(2)
end
end
end
...@@ -20,6 +20,16 @@ describe BitbucketServer::Paginator do ...@@ -20,6 +20,16 @@ describe BitbucketServer::Paginator do
expect { paginator.items }.to raise_error(StopIteration) expect { paginator.items }.to raise_error(StopIteration)
end end
it 'obeys limits' do
limited = described_class.new(connection, 'http://more-data', :pull_request, page_offset: 0, limit: 1)
allow(limited).to receive(:fetch_next_page).and_return(first_page)
expect(limited.has_next_page?).to be_truthy
expect(limited.items).to match(['item_1'])
expect(limited.has_next_page?).to be_truthy
expect { limited.items }.to raise_error(StopIteration)
end
it 'calls the connection with different offsets' do it 'calls the connection with different offsets' do
expect(connection).to receive(:get).with('http://more-data', start: 0, limit: BitbucketServer::Paginator::PAGE_LENGTH).and_return(page_attrs) expect(connection).to receive(:get).with('http://more-data', start: 0, limit: BitbucketServer::Paginator::PAGE_LENGTH).and_return(page_attrs)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment