Refactoring Github import

parent a7cb336e
...@@ -27,12 +27,13 @@ module Github ...@@ -27,12 +27,13 @@ module Github
self.reset_callbacks :validate self.reset_callbacks :validate
end end
attr_reader :project, :repository, :options, :cached_label_ids, attr_reader :project, :repository, :repo, :options, :errors,
:cached_gitlab_users, :cached_user_ids, :errors :cached_label_ids, :cached_gitlab_users, :cached_user_ids
def initialize(project, options) def initialize(project, options)
@project = project @project = project
@repository = project.repository @repository = project.repository
@repo = project.import_source
@options = options @options = options
@cached_label_ids = {} @cached_label_ids = {}
@cached_user_ids = {} @cached_user_ids = {}
...@@ -40,19 +41,32 @@ module Github ...@@ -40,19 +41,32 @@ module Github
@errors = [] @errors = []
end end
def execute(owner, repo) def execute
# Fetch repository fetch_repository
fetch_labels
fetch_milestones
fetch_pull_requests
fetch_issues
expire_repository_cache
errors
end
private
def fetch_repository
begin begin
project.create_repository project.create_repository
project.repository.add_remote('github', "https://{token}@github.com/#{owner}/#{repo}.git") project.repository.add_remote('github', "https://{token}@github.com/#{repo}.git")
project.repository.set_remote_as_mirror('github') project.repository.set_remote_as_mirror('github')
project.repository.fetch_remote('github', forced: true) project.repository.fetch_remote('github', forced: true)
rescue Gitlab::Shell::Error => e rescue Gitlab::Shell::Error => e
error(:project, "https://github.com/#{owner}/#{repo}.git", e.message) error(:project, "https://github.com/#{repo}.git", e.message)
end end
end
# Fetch labels def fetch_labels
url = "/repos/#{owner}/#{repo}/labels" url = "/repos/#{repo}/labels"
while url while url
response = Github::Client.new(options).get(url) response = Github::Client.new(options).get(url)
...@@ -72,13 +86,13 @@ module Github ...@@ -72,13 +86,13 @@ module Github
end end
# Cache labels # Cache labels
# TODO: we should take group labels in account
project.labels.select(:id, :title).find_each do |label| project.labels.select(:id, :title).find_each do |label|
@cached_label_ids[label.title] = label.id @cached_label_ids[label.title] = label.id
end end
end
# Fetch milestones def fetch_milestones
url = "/repos/#{owner}/#{repo}/milestones" url = "/repos/#{repo}/milestones"
while url while url
response = Github::Client.new(options).get(url, state: :all) response = Github::Client.new(options).get(url, state: :all)
...@@ -104,9 +118,10 @@ module Github ...@@ -104,9 +118,10 @@ module Github
url = response.rels[:next] url = response.rels[:next]
end end
end
# Fetch pull requests def fetch_pull_requests
url = "/repos/#{owner}/#{repo}/pulls" url = "/repos/#{repo}/pulls"
while url while url
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
...@@ -141,14 +156,13 @@ module Github ...@@ -141,14 +156,13 @@ module Github
merge_request.merge_request_diffs.create merge_request.merge_request_diffs.create
# Fetch review comments # Fetch review comments
review_comments_url = "/repos/#{owner}/#{repo}/pulls/#{pull_request.iid}/comments" review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments"
fetch_comments(merge_request, :review_comment, review_comments_url) fetch_comments(merge_request, :review_comment, review_comments_url)
# Fetch comments # Fetch comments
comments_url = "/repos/#{owner}/#{repo}/issues/#{pull_request.iid}/comments" comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments"
fetch_comments(merge_request, :comment, comments_url) fetch_comments(merge_request, :comment, comments_url)
rescue => e rescue => e
error(:pull_request, pull_request.url, e.message)
ensure ensure
clean_up_restored_branches(pull_request) clean_up_restored_branches(pull_request)
end end
...@@ -156,9 +170,10 @@ module Github ...@@ -156,9 +170,10 @@ module Github
url = response.rels[:next] url = response.rels[:next]
end end
end
# Fetch issues def fetch_issues
url = "/repos/#{owner}/#{repo}/issues" url = "/repos/#{repo}/issues"
while url while url
response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
...@@ -196,7 +211,7 @@ module Github ...@@ -196,7 +211,7 @@ module Github
if representation.has_comments? if representation.has_comments?
# Fetch comments # Fetch comments
comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments"
fetch_comments(issue, :comment, comments_url) fetch_comments(issue, :comment, comments_url)
end end
end end
...@@ -207,14 +222,8 @@ module Github ...@@ -207,14 +222,8 @@ module Github
url = response.rels[:next] url = response.rels[:next]
end end
repository.expire_content_cache
errors
end end
private
def fetch_comments(noteable, type, url) def fetch_comments(noteable, type, url)
while url while url
comments = Github::Client.new(options).get(url) comments = Github::Client.new(options).get(url)
...@@ -310,6 +319,10 @@ module Github ...@@ -310,6 +319,10 @@ module Github
"*Created by: #{author.username}*\n\n#{body}" "*Created by: #{author.username}*\n\n#{body}"
end end
def expire_repository_cache
repository.expire_content_cache
end
def error(type, url, message) def error(type, url, message)
errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message }
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment