Skip to content

G
gitlab-ce
Commit 6e242e82 authored by Yorick Peterse's avatar Yorick Peterse
Browse files
Options

Replace old GH importer with the parallel importer

parent 4dfe26cd
Hide whitespace changes
Inline Side-by-side
Showing with 23 additions and 1042 deletions
app/services/projects/import_service.rb
View file @ 6e242e82
...@@ -87,6 +87,10 @@ module Projects ...@@ -87,6 +87,10 @@ module Projects
end end
end end
def importer_class
Gitlab::ImportSources.importer(project.import_type)
end
def has_importer? def has_importer?
Gitlab::ImportSources.importer_names.include?(project.import_type) Gitlab::ImportSources.importer_names.include?(project.import_type)
end end
...@@ -95,10 +99,6 @@ module Projects ...@@ -95,10 +99,6 @@ module Projects
importer_class.new(project) importer_class.new(project)
end end
def importer_class
Gitlab::ImportSources.importer(project.import_type)
end
def unknown_url? def unknown_url?
project.import_url == Project::UNKNOWN_IMPORT_URL project.import_url == Project::UNKNOWN_IMPORT_URL
end end
......
lib/github/client.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Client
TIMEOUT = 60
DEFAULT_PER_PAGE = 100
attr_reader :connection, :rate_limit
def initialize(options)
@connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday|
faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT)
faraday.options.timeout = options.fetch(:timeout, TIMEOUT)
faraday.authorization 'token', options.fetch(:token)
faraday.adapter :net_http
faraday.ssl.verify = verify_ssl
end
@rate_limit = RateLimit.new(connection)
end
def get(url, query = {})
exceed, reset_in = rate_limit.get
sleep reset_in if exceed
Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query)))
end
private
def root_endpoint
custom_endpoint || github_endpoint
end
def custom_endpoint
github_omniauth_provider.dig('args', 'client_options', 'site')
end
def verify_ssl
# If there is no config, we're connecting to github.com
# and we should verify ssl.
github_omniauth_provider.fetch('verify_ssl', true)
end
def github_endpoint
OmniAuth::Strategies::GitHub.default_options[:client_options][:site]
end
def github_omniauth_provider
@github_omniauth_provider ||=
Gitlab.config.omniauth.providers
.find { |provider| provider.name == 'github' }
.to_h
end
end
end
lib/github/collection.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Collection
attr_reader :options
def initialize(options)
@options = options
end
def fetch(url, query = {})
return [] if url.blank?
Enumerator.new do |yielder|
loop do
response = client.get(url, query)
response.body.each { |item| yielder << item }
raise StopIteration unless response.rels.key?(:next)
url = response.rels[:next]
end
end.lazy
end
private
def client
@client ||= Github::Client.new(options)
end
end
end
lib/github/error.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
RepositoryFetchError = Class.new(StandardError)
end
lib/github/import.rb deleted 100644 → 0
View file @ 4dfe26cd
require_relative 'error'
require_relative 'import/issue'
require_relative 'import/legacy_diff_note'
require_relative 'import/merge_request'
require_relative 'import/note'
module Github
class Import
include Gitlab::ShellAdapter
attr_reader :project, :repository, :repo, :repo_url, :wiki_url,
:options, :errors, :cached, :verbose, :last_fetched_at
def initialize(project, options = {})
@project = project
@repository = project.repository
@repo = project.import_source
@repo_url = project.import_url
@wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
@options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user))
@verbose = options.fetch(:verbose, false)
@cached = Hash.new { |hash, key| hash[key] = Hash.new }
@errors = []
@last_fetched_at = nil
end
# rubocop: disable Rails/Output
def execute
puts 'Fetching repository...'.color(:aqua) if verbose
setup_and_fetch_repository
puts 'Fetching labels...'.color(:aqua) if verbose
fetch_labels
puts 'Fetching milestones...'.color(:aqua) if verbose
fetch_milestones
puts 'Fetching pull requests...'.color(:aqua) if verbose
fetch_pull_requests
puts 'Fetching issues...'.color(:aqua) if verbose
fetch_issues
puts 'Fetching releases...'.color(:aqua) if verbose
fetch_releases
puts 'Cloning wiki repository...'.color(:aqua) if verbose
fetch_wiki_repository
puts 'Expiring repository cache...'.color(:aqua) if verbose
expire_repository_cache
errors.empty?
rescue Github::RepositoryFetchError
expire_repository_cache
false
ensure
keep_track_of_errors
end
private
def setup_and_fetch_repository
begin
project.ensure_repository
project.repository.add_remote('github', repo_url)
project.repository.set_import_remote_as_mirror('github')
project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head')
fetch_remote(forced: true)
rescue Gitlab::Git::Repository::NoRepository,
Gitlab::Git::RepositoryMirroring::RemoteError,
Gitlab::Shell::Error => e
error(:project, repo_url, e.message)
raise Github::RepositoryFetchError
end
end
def fetch_remote(forced: false)
@last_fetched_at = Time.now
project.repository.fetch_remote('github', forced: forced)
end
def fetch_wiki_repository
return if project.wiki.repository_exists?
wiki_path = project.wiki.disk_path
gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
rescue Gitlab::Shell::Error => e
# GitHub error message when the wiki repo has not been created,
# this means that repo has wiki enabled, but have no pages. So,
# we can skip the import.
if e.message !~ /repository not exported/
error(:wiki, wiki_url, e.message)
end
end
def fetch_labels
url = "/repos/#{repo}/labels"
while url
response = Github::Client.new(options).get(url)
response.body.each do |raw|
begin
representation = Github::Representation::Label.new(raw)
label = project.labels.find_or_create_by!(title: representation.title) do |label|
label.color = representation.color
end
cached[:label_ids][representation.title] = label.id
rescue => e
error(:label, representation.url, e.message)
end
end
url = response.rels[:next]
end
end
def fetch_milestones
url = "/repos/#{repo}/milestones"
while url
response = Github::Client.new(options).get(url, state: :all)
response.body.each do |raw|
begin
milestone = Github::Representation::Milestone.new(raw)
next if project.milestones.where(iid: milestone.iid).exists?
project.milestones.create!(
iid: milestone.iid,
title: milestone.title,
description: milestone.description,
due_date: milestone.due_date,
state: milestone.state,
created_at: milestone.created_at,
updated_at: milestone.updated_at
)
rescue => e
error(:milestone, milestone.url, e.message)
end
end
url = response.rels[:next]
end
end
def fetch_pull_requests
url = "/repos/#{repo}/pulls"
while url
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
response.body.each do |raw|
pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project))
merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id)
next unless merge_request.new_record? && pull_request.valid?
begin
# If the PR has been created/updated after we last fetched the
# remote, we fetch again to get the up-to-date refs.
fetch_remote if pull_request.updated_at > last_fetched_at
author_id = user_id(pull_request.author, project.creator_id)
description = format_description(pull_request.description, pull_request.author)
merge_request.attributes = {
iid: pull_request.iid,
title: pull_request.title,
description: description,
source_project: pull_request.source_project,
source_branch: pull_request.source_branch_name,
source_branch_sha: pull_request.source_branch_sha,
target_project: pull_request.target_project,
target_branch: pull_request.target_branch_name,
target_branch_sha: pull_request.target_branch_sha,
state: pull_request.state,
milestone_id: milestone_id(pull_request.milestone),
author_id: author_id,
assignee_id: user_id(pull_request.assignee),
created_at: pull_request.created_at,
updated_at: pull_request.updated_at
}
merge_request.save!(validate: false)
merge_request.merge_request_diffs.create
review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments"
fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote)
rescue => e
error(:pull_request, pull_request.url, e.message)
end
end
url = response.rels[:next]
end
end
def fetch_issues
url = "/repos/#{repo}/issues"
while url
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
response.body.each { |raw| populate_issue(raw) }
url = response.rels[:next]
end
end
def populate_issue(raw)
representation = Github::Representation::Issue.new(raw, options)
begin
# Every pull request is an issue, but not every issue
# is a pull request. For this reason, "shared" actions
# for both features, like manipulating assignees, labels
# and milestones, are provided within the Issues API.
if representation.pull_request?
return unless representation.labels? || representation.comments?
merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid)
if representation.labels?
merge_request.update_attribute(:label_ids, label_ids(representation.labels))
end
fetch_comments_conditionally(merge_request, representation)
else
return if Issue.exists?(iid: representation.iid, project_id: project.id)
author_id = user_id(representation.author, project.creator_id)
issue = Issue.new
issue.iid = representation.iid
issue.project_id = project.id
issue.title = representation.title
issue.description = format_description(representation.description, representation.author)
issue.state = representation.state
issue.milestone_id = milestone_id(representation.milestone)
issue.author_id = author_id
issue.created_at = representation.created_at
issue.updated_at = representation.updated_at
issue.save!(validate: false)
issue.update(
label_ids: label_ids(representation.labels),
assignee_ids: assignee_ids(representation.assignees))
fetch_comments_conditionally(issue, representation)
end
rescue => e
error(:issue, representation.url, e.message)
end
end
def fetch_comments_conditionally(issuable, representation)
if representation.comments?
comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments"
fetch_comments(issuable, :comment, comments_url)
end
end
def fetch_comments(noteable, type, url, klass = Note)
while url
comments = Github::Client.new(options).get(url)
ActiveRecord::Base.no_touching do
comments.body.each do |raw|
begin
representation = Github::Representation::Comment.new(raw, options)
author_id = user_id(representation.author, project.creator_id)
note = klass.new
note.project_id = project.id
note.noteable = noteable
note.note = format_description(representation.note, representation.author)
note.commit_id = representation.commit_id
note.line_code = representation.line_code
note.author_id = author_id
note.created_at = representation.created_at
note.updated_at = representation.updated_at
note.save!(validate: false)
rescue => e
error(type, representation.url, e.message)
end
end
end
url = comments.rels[:next]
end
end
def fetch_releases
url = "/repos/#{repo}/releases"
while url
response = Github::Client.new(options).get(url)
response.body.each do |raw|
representation = Github::Representation::Release.new(raw)
next unless representation.valid?
release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
next unless release.new_record?
begin
release.description = representation.description
release.created_at = representation.created_at
release.updated_at = representation.updated_at
release.save!(validate: false)
rescue => e
error(:release, representation.url, e.message)
end
end
url = response.rels[:next]
end
end
def label_ids(labels)
labels.map { |label| cached[:label_ids][label.title] }.compact
end
def assignee_ids(assignees)
assignees.map { |assignee| user_id(assignee) }.compact
end
def milestone_id(milestone)
return unless milestone.present?
project.milestones.select(:id).find_by(iid: milestone.iid)&.id
end
def user_id(user, fallback_id = nil)
return unless user.present?
return cached[:user_ids][user.id] if cached[:user_ids][user.id].present?
gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
cached[:gitlab_user_ids][user.id] = gitlab_user_id.present?
cached[:user_ids][user.id] = gitlab_user_id || fallback_id
end
def user_id_by_email(email)
return nil unless email
::User.find_by_any_email(email)&.id
end
def user_id_by_external_uid(id)
return nil unless id
::User.select(:id)
.joins(:identities)
.merge(::Identity.where(provider: :github, extern_uid: id))
.first&.id
end
def format_description(body, author)
return body if cached[:gitlab_user_ids][author.id]
"*Created by: #{author.username}*\n\n#{body}"
end
def expire_repository_cache
repository.expire_content_cache if project.repository_exists?
end
def keep_track_of_errors
return unless errors.any?
project.update_column(:import_error, {
message: 'The remote data could not be fully imported.',
errors: errors
}.to_json)
end
def error(type, url, message)
errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message }
end
end
end
lib/github/import/issue.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Import
class Issue < ::Issue
self.table_name = 'issues'
self.reset_callbacks :save
self.reset_callbacks :create
self.reset_callbacks :commit
self.reset_callbacks :update
self.reset_callbacks :validate
end
end
end
lib/github/import/legacy_diff_note.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Import
class LegacyDiffNote < ::LegacyDiffNote
self.table_name = 'notes'
self.store_full_sti_class = false
self.reset_callbacks :commit
self.reset_callbacks :update
self.reset_callbacks :validate
end
end
end
lib/github/import/merge_request.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Import
class MergeRequest < ::MergeRequest
self.table_name = 'merge_requests'
self.reset_callbacks :create
self.reset_callbacks :save
self.reset_callbacks :commit
self.reset_callbacks :update
self.reset_callbacks :validate
end
end
end
lib/github/import/note.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Import
class Note < ::Note
self.table_name = 'notes'
self.store_full_sti_class = false
self.reset_callbacks :save
self.reset_callbacks :commit
self.reset_callbacks :update
self.reset_callbacks :validate
end
end
end
lib/github/rate_limit.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class RateLimit
SAFE_REMAINING_REQUESTS = 100
SAFE_RESET_TIME = 500
RATE_LIMIT_URL = '/rate_limit'.freeze
attr_reader :connection
def initialize(connection)
@connection = connection
end
def get
response = connection.get(RATE_LIMIT_URL)
# GitHub Rate Limit API returns 404 when the rate limit is disabled
return false unless response.status != 404
body = Oj.load(response.body, class_cache: false, mode: :compat)
remaining = body.dig('rate', 'remaining').to_i
reset_in = body.dig('rate', 'reset').to_i
exceed = remaining <= SAFE_REMAINING_REQUESTS
[exceed, reset_in]
end
end
end
lib/github/repositories.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Repositories
attr_reader :options
def initialize(options)
@options = options
end
def fetch
Collection.new(options).fetch(repos_url)
end
private
def repos_url
'/user/repos'
end
end
end
lib/github/representation/base.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Base
def initialize(raw, options = {})
@raw = raw
@options = options
end
def id
raw['id']
end
def url
raw['url']
end
def created_at
raw['created_at']
end
def updated_at
raw['updated_at']
end
private
attr_reader :raw, :options
end
end
end
lib/github/representation/branch.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Branch < Representation::Base
attr_reader :repository
def user
raw.dig('user', 'login') || 'unknown'
end
def repo?
raw['repo'].present?
end
def repo
return unless repo?
@repo ||= Github::Representation::Repo.new(raw['repo'])
end
def ref
raw['ref']
end
def sha
raw['sha']
end
def short_sha
Commit.truncate_sha(sha)
end
def valid?
sha.present? && ref.present?
end
def restore!(name)
repository.create_branch(name, sha)
rescue Gitlab::Git::Repository::InvalidRef => e
Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}")
end
def remove!(name)
repository.delete_branch(name)
rescue Gitlab::Git::Repository::DeleteBranchError => e
Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}")
end
private
def repository
@repository ||= options.fetch(:repository)
end
end
end
end
lib/github/representation/comment.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Comment < Representation::Base
def note
raw['body'] || ''
end
def author
@author ||= Github::Representation::User.new(raw['user'], options)
end
def commit_id
raw['commit_id']
end
def line_code
return unless on_diff?
parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines)
generate_line_code(parsed_lines.to_a.last)
end
private
def generate_line_code(line)
Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos)
end
def on_diff?
diff_hunk.present?
end
def diff_hunk
raw['diff_hunk']
end
def file_path
raw['path']
end
end
end
end
lib/github/representation/issuable.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Issuable < Representation::Base
def iid
raw['number']
end
def title
raw['title']
end
def description
raw['body'] || ''
end
def milestone
return unless raw['milestone'].present?
@milestone ||= Github::Representation::Milestone.new(raw['milestone'])
end
def author
@author ||= Github::Representation::User.new(raw['user'], options)
end
def labels?
raw['labels'].any?
end
def labels
@labels ||= Array(raw['labels']).map do |label|
Github::Representation::Label.new(label, options)
end
end
end
end
end
lib/github/representation/issue.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Issue < Representation::Issuable
def state
raw['state'] == 'closed' ? 'closed' : 'opened'
end
def comments?
raw['comments'] > 0
end
def pull_request?
raw['pull_request'].present?
end
def assigned?
raw['assignees'].present?
end
def assignees
@assignees ||= Array(raw['assignees']).map do |user|
Github::Representation::User.new(user, options)
end
end
end
end
end
lib/github/representation/label.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Label < Representation::Base
def color
"##{raw['color']}"
end
def title
raw['name']
end
end
end
end
lib/github/representation/milestone.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Milestone < Representation::Base
def iid
raw['number']
end
def title
raw['title']
end
def description
raw['description']
end
def due_date
raw['due_on']
end
def state
raw['state'] == 'closed' ? 'closed' : 'active'
end
end
end
end
lib/github/representation/pull_request.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class PullRequest < Representation::Issuable
delegate :sha, to: :source_branch, prefix: true
delegate :sha, to: :target_branch, prefix: true
def source_project
project
end
def source_branch_name
# Mimic the "user:branch" displayed in the MR widget,
# i.e. "Request to merge rymai:add-external-mounts into master"
cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref
end
def target_project
project
end
def target_branch_name
target_branch.ref
end
def state
return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present?
return 'closed' if raw['state'] == 'closed'
'opened'
end
def opened?
state == 'opened'
end
def valid?
source_branch.valid? && target_branch.valid?
end
def assigned?
raw['assignee'].present?
end
def assignee
return unless assigned?
@assignee ||= Github::Representation::User.new(raw['assignee'], options)
end
private
def project
@project ||= options.fetch(:project)
end
def source_branch
@source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository)
end
def target_branch
@target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository)
end
def cross_project?
return true unless source_branch.repo?
source_branch.repo.id != target_branch.repo.id
end
end
end
end
lib/github/representation/release.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Release < Representation::Base
def description
raw['body']
end
def tag
raw['tag_name']
end
def valid?
!raw['draft']
end
end
end
end
lib/github/representation/repo.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class Repo < Representation::Base
end
end
end
lib/github/representation/user.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
module Representation
class User < Representation::Base
def email
return @email if defined?(@email)
@email = Github::User.new(username, options).get.fetch('email', nil)
end
def username
raw['login']
end
end
end
end
lib/github/response.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class Response
attr_reader :raw, :headers, :status
def initialize(response)
@raw = response
@headers = response.headers
@status = response.status
end
def body
Oj.load(raw.body, class_cache: false, mode: :compat)
end
def rels
links = headers['Link'].to_s.split(', ').map do |link|
href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures
[name.to_sym, href]
end
Hash[*links.flatten]
end
end
end
lib/github/user.rb deleted 100644 → 0
View file @ 4dfe26cd
module Github
class User
attr_reader :username, :options
def initialize(username, options)
@username = username
@options = options
end
def get
client.get(user_url).body
end
private
def client
@client ||= Github::Client.new(options)
end
def user_url
"/users/#{username}"
end
end
end
lib/gitlab/import_sources.rb
View file @ 6e242e82
...@@ -8,7 +8,7 @@ module Gitlab ...@@ -8,7 +8,7 @@ module Gitlab
ImportSource = Struct.new(:name, :title, :importer) ImportSource = Struct.new(:name, :title, :importer)
ImportTable = [ ImportTable = [
ImportSource.new('github', 'GitHub', Github::Import), ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter),
ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer),
ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer),
ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer),
......
lib/tasks/import.rake
View file @ 6e242e82
...@@ -7,7 +7,7 @@ class GithubImport ...@@ -7,7 +7,7 @@ class GithubImport
end end
def initialize(token, gitlab_username, project_path, extras) def initialize(token, gitlab_username, project_path, extras)
@options = { token: token, verbose: true } @options = { token: token }
@project_path = project_path @project_path = project_path
@current_user = User.find_by_username(gitlab_username) @current_user = User.find_by_username(gitlab_username)
@github_repo = extras.empty? ? nil : extras.first @github_repo = extras.empty? ? nil : extras.first
...@@ -42,7 +42,9 @@ class GithubImport ...@@ -42,7 +42,9 @@ class GithubImport
import_success = false import_success = false
timings = Benchmark.measure do timings = Benchmark.measure do
import_success = Github::Import.new(@project, @options).execute import_success = Gitlab::GithubImport::SequentialImporter
.new(@project, token: @options[:token])
.execute
end end
if import_success if import_success
......
spec/lib/github/client_spec.rb deleted 100644 → 0
View file @ 4dfe26cd
require 'spec_helper'
describe Github::Client do
let(:connection) { spy }
let(:rate_limit) { double(get: [false, 1]) }
let(:client) { described_class.new({}) }
let(:results) { double }
let(:response) { double }
before do
allow(Faraday).to receive(:new).and_return(connection)
allow(Github::RateLimit).to receive(:new).with(connection).and_return(rate_limit)
end
describe '#get' do
before do
allow(Github::Response).to receive(:new).with(results).and_return(response)
end
it 'uses a default per_page param' do
expect(connection).to receive(:get).with('/foo', per_page: 100).and_return(results)
expect(client.get('/foo')).to eq(response)
end
context 'with per_page given' do
it 'overwrites the default per_page' do
expect(connection).to receive(:get).with('/foo', per_page: 30).and_return(results)
expect(client.get('/foo', per_page: 30)).to eq(response)
end
end
end
end
spec/lib/github/import/legacy_diff_note_spec.rb deleted 100644 → 0
View file @ 4dfe26cd
require 'spec_helper'
describe Github::Import::LegacyDiffNote do
describe '#type' do
it 'returns the original note type' do
expect(described_class.new.type).to eq('LegacyDiffNote')
end
end
end
spec/lib/github/import/note_spec.rb deleted 100644 → 0
View file @ 4dfe26cd
require 'spec_helper'
describe Github::Import::Note do
describe '#type' do
it 'returns the original note type' do
expect(described_class.new.type).to eq('Note')
end
end
end
spec/lib/gitlab/import_sources_spec.rb
View file @ 6e242e82
...@@ -56,7 +56,7 @@ describe Gitlab::ImportSources do ...@@ -56,7 +56,7 @@ describe Gitlab::ImportSources do
describe '.importer' do describe '.importer' do
import_sources = { import_sources = {
'github' => Github::Import, 'github' => Gitlab::GithubImport::ParallelImporter,
'bitbucket' => Gitlab::BitbucketImport::Importer, 'bitbucket' => Gitlab::BitbucketImport::Importer,
'gitlab' => Gitlab::GitlabImport::Importer, 'gitlab' => Gitlab::GitlabImport::Importer,
'google_code' => Gitlab::GoogleCodeImport::Importer, 'google_code' => Gitlab::GoogleCodeImport::Importer,
......
spec/services/projects/import_service_spec.rb
View file @ 6e242e82
...@@ -72,21 +72,24 @@ describe Projects::ImportService do ...@@ -72,21 +72,24 @@ describe Projects::ImportService do
end end
context 'with a Github repository' do context 'with a Github repository' do
it 'succeeds if repository import is successfully' do it 'succeeds if repository import was scheduled' do
expect_any_instance_of(Github::Import).to receive(:execute).and_return(true) expect_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
.and_return(true)
result = subject.execute result = subject.execute
expect(result[:status]).to eq :success expect(result[:status]).to eq :success
end end
it 'fails if repository import fails' do it 'fails if repository import was not scheduled' do
expect_any_instance_of(Repository).to receive(:fetch_remote).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) expect_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
.and_return(false)
result = subject.execute result = subject.execute
expect(result[:status]).to eq :error expect(result[:status]).to eq :error
expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.path_with_namespace} - The remote data could not be imported."
end end
end end
...@@ -127,47 +130,22 @@ describe Projects::ImportService do ...@@ -127,47 +130,22 @@ describe Projects::ImportService do
end end
it 'succeeds if importer succeeds' do it 'succeeds if importer succeeds' do
allow_any_instance_of(Github::Import).to receive(:execute).and_return(true) allow_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute).and_return(true)
result = subject.execute result = subject.execute
expect(result[:status]).to eq :success expect(result[:status]).to eq :success
end end
it 'flushes various caches' do
allow_any_instance_of(Github::Import).to receive(:execute)
.and_return(true)
expect_any_instance_of(Repository).to receive(:expire_content_cache)
subject.execute
end
it 'fails if importer fails' do it 'fails if importer fails' do
allow_any_instance_of(Github::Import).to receive(:execute).and_return(false) allow_any_instance_of(Gitlab::GithubImport::ParallelImporter)
.to receive(:execute)
result = subject.execute .and_return(false)
expect(result[:status]).to eq :error
expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - The remote data could not be imported."
end
it 'fails if importer raise an error' do
allow_any_instance_of(Github::Import).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API'))
result = subject.execute result = subject.execute
expect(result[:status]).to eq :error expect(result[:status]).to eq :error
expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - Github: failed to connect API"
end
it 'expires content cache after error' do
allow_any_instance_of(Project).to receive(:repository_exists?).and_return(false)
expect_any_instance_of(Repository).to receive(:fetch_remote).and_raise(Gitlab::Shell::Error.new)
expect_any_instance_of(Repository).to receive(:expire_content_cache)
subject.execute
end end
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7