Commit ad362640 authored by Illya Klymov's avatar Illya Klymov

Implement JSON status for importers behind feature flag

- introduce new_import_ui feature flag
- implement specialized serializers
parent d80cb6f9
# frozen_string_literal: true
class Import::BaseController < ApplicationController
include ActionView::Helpers::SanitizeHelper
before_action :import_rate_limit, only: [:create]
def status
respond_to do |format|
format.json do
render json: { imported_projects: serialized_imported_projects,
provider_repos: serialized_provider_repos,
incompatible_repos: serialized_incompatible_repos,
namespaces: serialized_namespaces }
end
format.html
end
end
def realtime_changes
Gitlab::PollingInterval.set_header(response, interval: 3_000)
render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
end
protected
def importable_repos
raise NotImplementedError
end
def incompatible_repos
[]
end
def provider_name
raise NotImplementedError
end
def provider_url
raise NotImplementedError
end
private
def sanitized_filter_param
@filter ||= sanitize(params[:filter])
end
def filtered(collection)
return collection unless sanitized_filter_param
collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
end
def serialized_provider_repos
Import::ProviderRepoSerializer.new(current_user: current_user).represent(importable_repos, provider: provider_name, provider_url: provider_url)
end
def serialized_incompatible_repos
Import::ProviderRepoSerializer.new(current_user: current_user).represent(incompatible_repos, provider: provider_name, provider_url: provider_url)
end
def serialized_imported_projects
ProjectSerializer.new.represent(already_added_projects, serializer: :import, provider_url: provider_url)
end
def already_added_projects
@already_added_projects ||= filtered(find_already_added_projects(provider_name))
end
def serialized_namespaces
NamespaceSerializer.new.represent(namespaces)
end
def already_added_project_names
@already_added_projects_names ||= already_added_projects.pluck(:import_source) # rubocop:disable CodeReuse/ActiveRecord
end
def namespaces
current_user.manageable_groups_with_routes
end
# rubocop: disable CodeReuse/ActiveRecord
def find_already_added_projects(import_type)
current_user.created_projects.where(import_type: import_type).with_import_state
......
# frozen_string_literal: true
class Import::BitbucketController < Import::BaseController
extend ::Gitlab::Utils::Override
include ActionView::Helpers::SanitizeHelper
before_action :verify_bitbucket_import_enabled
......@@ -10,7 +12,7 @@ class Import::BitbucketController < Import::BaseController
rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized
def callback
response = client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
response = oauth_client.auth_code.get_token(params[:code], redirect_uri: users_import_bitbucket_callback_url)
session[:bitbucket_token] = response.token
session[:bitbucket_expires_at] = response.expires_at
......@@ -22,9 +24,12 @@ class Import::BitbucketController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord
def status
if Feature.enabled?(:new_import_ui)
return super
end
bitbucket_client = Bitbucket::Client.new(credentials)
repos = bitbucket_client.repos(filter: sanitized_filter_param)
@repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
@already_added_projects = find_already_added_projects('bitbucket')
......@@ -38,6 +43,10 @@ class Import::BitbucketController < Import::BaseController
render json: find_jobs('bitbucket')
end
def realtime_changes
super
end
def create
bitbucket_client = Bitbucket::Client.new(credentials)
......@@ -59,7 +68,7 @@ class Import::BitbucketController < Import::BaseController
project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
render json: ProjectSerializer.new.represent(project)
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
......@@ -68,16 +77,50 @@ class Import::BitbucketController < Import::BaseController
end
end
protected
# rubocop: disable CodeReuse/ActiveRecord
override :importable_repos
def importable_repos
already_added_projects_names = already_added_projects.pluck(:import_source)
bitbucket_repos.reject { |repo| (already_added_projects_names.include? repo.full_name) || !repo.valid? }
end
# rubocop: enable CodeReuse/ActiveRecord
override :incompatible_repos
def incompatible_repos
bitbucket_repos.reject { |repo| repo.valid? }
end
override :provider_name
def provider_name
:bitbucket
end
override :provider_url
def provider_url
provider.url
end
private
def client
@client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
def oauth_client
@oauth_client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
end
def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
end
def client
@client ||= Bitbucket::Client.new(credentials)
end
def bitbucket_repos
@bitbucket_repos ||= client.repos(filter: sanitized_filter_param).to_a
end
def options
OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
end
......@@ -91,7 +134,7 @@ class Import::BitbucketController < Import::BaseController
end
def go_to_bitbucket_for_permissions
redirect_to client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
redirect_to oauth_client.auth_code.authorize_url(redirect_uri: users_import_bitbucket_callback_url)
end
def bitbucket_unauthorized
......
# frozen_string_literal: true
class Import::BitbucketServerController < Import::BaseController
extend ::Gitlab::Utils::Override
include ActionView::Helpers::SanitizeHelper
before_action :verify_bitbucket_server_import_enabled
......@@ -24,7 +26,7 @@ class Import::BitbucketServerController < Import::BaseController
end
def create
repo = bitbucket_client.repo(@project_key, @repo_slug)
repo = client.repo(@project_key, @repo_slug)
unless repo
return render json: { errors: _("Project %{project_repo} could not be found") % { project_repo: "#{@project_key}/#{@repo_slug}" } }, status: :unprocessable_entity
......@@ -38,7 +40,7 @@ class Import::BitbucketServerController < Import::BaseController
project = Gitlab::BitbucketServerImport::ProjectCreator.new(@project_key, @repo_slug, repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
render json: ProjectSerializer.new.represent(project)
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
......@@ -59,7 +61,16 @@ class Import::BitbucketServerController < Import::BaseController
# rubocop: disable CodeReuse/ActiveRecord
def status
@collection = bitbucket_client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
if Feature.enabled?(:new_import_ui)
begin
return super
rescue BitbucketServer::Connection::ConnectionError => error
render json: { error: _("Unable to connect to server: %{error}") % { error: error } }, status: :unprocessable_entity
return
end
end
@collection = client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param)
@repos, @incompatible_repos = @collection.partition { |repo| repo.valid? }
# Use the import URL to filter beyond what BaseService#find_already_added_projects
......@@ -78,6 +89,38 @@ class Import::BitbucketServerController < Import::BaseController
render json: find_jobs('bitbucket_server')
end
def realtime_changes
super
end
protected
# rubocop: disable CodeReuse/ActiveRecord
override :importable_repos
def importable_repos
# Use the import URL to filter beyond what BaseService#find_already_added_projects
already_added_projects = filter_added_projects('bitbucket_server', bitbucket_repos.map(&:browse_url))
already_added_projects_names = already_added_projects.pluck(:import_source)
bitbucket_repos.reject { |repo| already_added_projects_names.include?(repo.browse_url) || !repo.valid? }
end
# rubocop: enable CodeReuse/ActiveRecord
override :incompatible_repos
def incompatible_repos
bitbucket_repos.reject { |repo| repo.valid? }
end
override :provider_name
def provider_name
:bitbucket_server
end
override :provider_url
def provider_url
session[bitbucket_server_url_key]
end
private
# rubocop: disable CodeReuse/ActiveRecord
......@@ -86,8 +129,12 @@ class Import::BitbucketServerController < Import::BaseController
end
# rubocop: enable CodeReuse/ActiveRecord
def bitbucket_client
@bitbucket_client ||= BitbucketServer::Client.new(credentials)
def client
@client ||= BitbucketServer::Client.new(credentials)
end
def bitbucket_repos
@bitbucket_repos ||= client.repos(page_offset: page_offset, limit: limit_per_page, filter: sanitized_filter_param).to_a
end
def validate_import_params
......
# frozen_string_literal: true
class Import::FogbugzController < Import::BaseController
extend ::Gitlab::Utils::Override
before_action :verify_fogbugz_import_enabled
before_action :user_map, only: [:new_user_map, :create_user_map]
before_action :verify_blocked_uri, only: :callback
......@@ -48,6 +50,10 @@ class Import::FogbugzController < Import::BaseController
return redirect_to new_import_fogbugz_path
end
if Feature.enabled?(:new_import_ui)
return super
end
@repos = client.repos
@already_added_projects = find_already_added_projects('fogbugz')
......@@ -57,6 +63,10 @@ class Import::FogbugzController < Import::BaseController
end
# rubocop: enable CodeReuse/ActiveRecord
def realtime_changes
super
end
def jobs
render json: find_jobs('fogbugz')
end
......@@ -69,12 +79,36 @@ class Import::FogbugzController < Import::BaseController
project = Gitlab::FogbugzImport::ProjectCreator.new(repo, fb_session, current_user.namespace, current_user, umap).execute
if project.persisted?
render json: ProjectSerializer.new.represent(project)
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
end
protected
# rubocop: disable CodeReuse/ActiveRecord
override :importable_repos
def importable_repos
repos = client.repos
already_added_projects = find_already_added_projects('fogbugz')
already_added_projects_names = already_added_projects.pluck(:import_source)
repos.reject { |repo| already_added_projects_names.include? repo.name }
end
# rubocop: enable CodeReuse/ActiveRecord
override :provider_name
def provider_name
:fogbugz
end
override :provider_url
def provider_url
session[:fogbugz_uri]
end
private
def client
......
......@@ -76,7 +76,7 @@ class Import::GithubController < Import::BaseController
def serialized_provider_repos
repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name }
ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
Import::ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
end
def serialized_namespaces
......
# frozen_string_literal: true
class Import::BaseProviderRepoEntity < Grape::Entity
expose :id
expose :full_name
expose :owner_name
expose :sanitized_name
expose :provider_link
end
# frozen_string_literal: true
class Import::BitbucketProviderRepoEntity < Import::BaseProviderRepoEntity
include ImportHelper
expose :id, override: true do |repo|
repo.full_name
end
expose :owner_name, override: true do |repo|
repo.owner
end
expose :sanitized_name, override: true do |repo|
repo.name.gsub(/[^\s\w.-]/, '')
end
expose :provider_link, override: true do |repo, options|
repo.clone_url
end
end
# frozen_string_literal: true
class Import::BitbucketServerProviderRepoEntity < Import::BitbucketProviderRepoEntity
expose :owner_name, override: true do
''
end
expose :provider_link, override: true do |repo, options|
repo.browse_url
end
end
# frozen_string_literal: true
class Import::FogbugzProviderRepoEntity < Import::BaseProviderRepoEntity
include ImportHelper
expose :full_name, override: true do |repo|
repo.name
end
expose :owner_name, override: true do
''
end
expose :sanitized_name, override: true do |repo|
repo.safe_name
end
expose :provider_link, override: true do |repo, options|
provider_project_link_url(options[:provider_url], repo.path)
end
end
# frozen_string_literal: true
class ProviderRepoEntity < Grape::Entity
class Import::GithubishProviderRepoEntity < Import::BaseProviderRepoEntity
include ImportHelper
expose :id
expose :full_name
expose :owner_name do |provider_repo, options|
expose :owner_name, override: true do |provider_repo, options|
owner_name(provider_repo, options[:provider])
end
expose :sanitized_name do |provider_repo|
expose :sanitized_name, override: true do |provider_repo|
sanitize_project_name(provider_repo[:name])
end
expose :provider_link do |provider_repo, options|
expose :provider_link, override: true do |provider_repo, options|
provider_project_link_url(options[:provider_url], provider_repo[:full_name])
end
......
# frozen_string_literal: true
class Import::ProviderRepoSerializer < BaseSerializer
def represent(repo, opts = {})
entity =
case opts[:provider]
when :fogbugz
Import::FogbugzProviderRepoEntity
when :github, :gitea
Import::GithubishProviderRepoEntity
when :bitbucket
Import::BitbucketProviderRepoEntity
when :bitbucket_server
Import::BitbucketServerProviderRepoEntity
else
raise NotImplementedError
end
super(repo, opts, entity)
end
end
# frozen_string_literal: true
class ProviderRepoSerializer < BaseSerializer
entity ProviderRepoEntity
end
......@@ -31,6 +31,7 @@ namespace :import do
get :status
get :callback
get :jobs
get :realtime_changes
end
resource :bitbucket_server, only: [:create, :new], controller: :bitbucket_server do
......@@ -38,6 +39,7 @@ namespace :import do
get :status
get :callback
get :jobs
get :realtime_changes
end
resource :google_code, only: [:create, :new], controller: :google_code do
......@@ -53,6 +55,7 @@ namespace :import do
get :status
post :callback
get :jobs
get :realtime_changes
get :new_user_map, path: :user_map
post :create_user_map, path: :user_map
......
......@@ -56,8 +56,20 @@ RSpec.describe Import::BitbucketController do
describe "GET status" do
before do
@repo = double(slug: 'vim', owner: 'asd', full_name: 'asd/vim', "valid?" => true)
@repo = double(name: 'vim', slug: 'vim', owner: 'asd', full_name: 'asd/vim', clone_url: "http://test.host/demo/url.git", "valid?" => true)
assign_session_tokens
stub_feature_flags(new_import_ui: false)
end
it_behaves_like 'import controller with new_import_ui feature flag' do
before do
allow(controller).to receive(:provider_url).and_return('http://demobitbucket.org')
end
let(:repo) { @repo }
let(:repo_id) { @repo.full_name }
let(:import_source) { @repo.full_name }
let(:provider_name) { 'bitbucket' }
end
it "assigns variables" do
......
......@@ -33,7 +33,7 @@ RSpec.describe Import::BitbucketServerController do
let(:project_name) { "my-project_123" }
before do
allow(controller).to receive(:bitbucket_client).and_return(client)
allow(controller).to receive(:client).and_return(client)
repo = double(name: project_name)
allow(client).to receive(:repo).with(project_key, repo_slug).and_return(repo)
assign_session_tokens
......@@ -139,12 +139,20 @@ RSpec.describe Import::BitbucketServerController do
let(:repos) { instance_double(BitbucketServer::Collection) }
before do
allow(controller).to receive(:bitbucket_client).and_return(client)
allow(controller).to receive(:client).and_return(client)
@repo = double(slug: 'vim', project_key: 'asd', full_name: 'asd/vim', "valid?" => true, project_name: 'asd', browse_url: 'http://test', name: 'vim')
@invalid_repo = double(slug: 'invalid', project_key: 'foobar', full_name: 'asd/foobar', "valid?" => false, browse_url: 'http://bad-repo')
@created_repo = double(slug: 'created', project_key: 'existing', full_name: 'group/created', "valid?" => true, browse_url: 'http://existing')
assign_session_tokens
stub_feature_flags(new_import_ui: false)
end
it_behaves_like 'import controller with new_import_ui feature flag' do
let(:repo) { @repo }
let(:repo_id) { @repo.full_name }
let(:import_source) { @repo.browse_url }
let(:provider_name) { 'bitbucket_server' }
end
it 'assigns repository categories' do
......
......@@ -80,8 +80,16 @@ RSpec.describe Import::FogbugzController do
describe 'GET status' do
before do
@repo = OpenStruct.new(name: 'vim')
@repo = OpenStruct.new(id: 'demo', name: 'vim')
stub_client(valid?: true)
stub_feature_flags(new_import_ui: false)
end
it_behaves_like 'import controller with new_import_ui feature flag' do
let(:repo) { @repo }
let(:repo_id) { @repo.id }
let(:import_source) { @repo.name }
let(:provider_name) { 'fogbugz' }
end
it 'assigns variables' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Import::BitbucketProviderRepoEntity do
let(:repo_data) do
{
"name" => "repo_name",
"full_name" => "owner/repo_name",
"links" => {
"clone" => [
{
"href" => "https://bitbucket.org/owner/repo_name",
"name" => "https"
}
]
}
}
end
let(:repo) { Bitbucket::Representation::Repo.new(repo_data) }
subject { described_class.new(repo).as_json }
it_behaves_like 'exposes required fields for import entity'
end
# frozen_string_literal: true
require 'spec_helper'
describe Import::BitbucketServerProviderRepoEntity do
let(:repo_data) do
{
"name" => "demo",
"project" => {
"name" => "demo"
},
"links" => {
"self" => [
{
"href" => "http://local.bitbucket.server/demo/demo.git",
"name" => "http"
}
]
}
}
end
let(:repo) { BitbucketServer::Representation::Repo.new(repo_data) }
subject { described_class.new(repo).as_json }
it_behaves_like 'exposes required fields for import entity'
end
# frozen_string_literal: true
require 'spec_helper'
describe Import::FogbugzProviderRepoEntity do
let(:repo_data) do
{
"ixProject" => "foo",
"sProject" => "demo"
}
end
let(:repo) { Gitlab::FogbugzImport::Repository.new(repo_data) }
subject { described_class.new(repo).as_json }
it_behaves_like 'exposes required fields for import entity'
end
# frozen_string_literal: true
require 'spec_helper'
describe Import::GithubishProviderRepoEntity do
let(:repo) do
{
id: 1,
full_name: 'full/name',
name: 'name',
owner: { login: 'owner' }
}
end
subject { described_class.new(repo).as_json }
it_behaves_like 'exposes required fields for import entity'
end
# frozen_string_literal: true
require 'spec_helper'
describe Import::ProviderRepoSerializer do
using RSpec::Parameterized::TableSyntax
describe '#represent' do
where(:provider, :class_name) do
:github | 'Import::GithubishProviderRepoEntity'
:gitea | 'Import::GithubishProviderRepoEntity'
:bitbucket | 'Import::BitbucketProviderRepoEntity'
:bitbucket_server | 'Import::BitbucketServerProviderRepoEntity'
:fogbugz | 'Import::FogbugzProviderRepoEntity'
end
with_them do
it 'uses correct entity class' do
opts = { provider: provider }
expect(class_name.constantize).to receive(:represent)
described_class.new.represent({}, opts)
end
end
it 'raises an error if invalid provider supplied' do
expect { described_class.new.represent({}, { provider: :invalid })}.to raise_error { NotImplementedError }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProviderRepoEntity do
include ImportHelper
let(:provider_repo) { { id: 1, full_name: 'full/name', name: 'name', owner: { login: 'owner' } } }
let(:provider) { :github }
let(:provider_url) { 'https://github.com' }
let(:entity) { described_class.represent(provider_repo, provider: provider, provider_url: provider_url) }
describe '#as_json' do
subject { entity.as_json }
it 'includes required fields' do
expect(subject[:id]).to eq(provider_repo[:id])
expect(subject[:full_name]).to eq(provider_repo[:full_name])
expect(subject[:owner_name]).to eq(provider_repo[:owner][:login])
expect(subject[:sanitized_name]).to eq(sanitize_project_name(provider_repo[:name]))
expect(subject[:provider_link]).to eq(provider_project_link_url(provider_url, provider_repo[:full_name]))
end
end
end
# frozen_string_literal: true
RSpec.shared_examples 'import controller with new_import_ui feature flag' do
include ImportSpecHelper
context 'with new_import_ui feature flag enabled' do
let(:group) { create(:group) }
before do
stub_feature_flags(new_import_ui: true)
group.add_owner(user)
end
it "returns variables for json request" do
project = create(:project, import_type: provider_name, creator_id: user.id)
stub_client(repos: [repo])
get :status, format: :json
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.dig("imported_projects", 0, "id")).to eq(project.id)
expect(json_response.dig("provider_repos", 0, "id")).to eq(repo_id)
expect(json_response.dig("namespaces", 0, "id")).to eq(group.id)
end
it "does not show already added project" do
project = create(:project, import_type: provider_name, namespace: user.namespace, import_status: :finished, import_source: import_source)
stub_client(repos: [repo], orgs: [])
get :status, format: :json
expect(json_response.dig("imported_projects", 0, "id")).to eq(project.id)
expect(json_response.dig("provider_repos")).to eq([])
end
end
end
# frozen_string_literal: true
RSpec.shared_examples 'exposes required fields for import entity' do
describe 'exposes required fields' do
it 'exposes id' do
expect(subject).to include(:id)
end
it 'exposes full name' do
expect(subject).to include(:full_name)
end
it 'exposes owner name' do
expect(subject).to include(:owner_name)
end
it 'exposes sanitized name' do
expect(subject).to include(:sanitized_name)
end
it 'exposes provider link' do
expect(subject).to include(:provider_link)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment