Merge branches 'feature/project-export-ui-experimental' and...

Merge branches 'feature/project-export-ui-experimental' and 'feature/project-import' of gitlab.com:gitlab-org/gitlab-ce into feature/project-export-ui-experimental

app/services/projects/import_export/export_service.rb

@@ -5,10 +5,7 @@ module Projects
       def execute(options = {})
         @shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work'))
         # TODO handle errors
-        save_project_tree
-        bundle_repo
-        bundle_wiki_repo
-        save_all
+        save_all if [save_project_tree, bundle_repo, bundle_wiki_repo].all?
       end
 
       private

lib/gitlab/import_export.rb

@@ -6,7 +6,7 @@ module Gitlab
       File.join(storage_path, relative_path)
     end
 
-    def project_atts
+    def project_attributes
       %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived)
     end
 

lib/gitlab/import_export/command_line_util.rb

 module Gitlab
   module ImportExport
     module CommandLineUtil
-      def tar_cf(archive:, dir:)
-        tar_with_options(archive: archive, dir: dir, options: 'cf')
+      def tar_czf(archive:, dir:)
+        tar_with_options(archive: archive, dir: dir, options: 'czf')
       end
 
       def untar_zxf(archive:, dir:)
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
-      def untar_xf(archive:, dir:)
-        untar_with_options(archive: archive, dir: dir, options: 'xf')
+      def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
       end
 
-      def tar_czf(archive:, dir:)
-        tar_with_options(archive: archive, dir: dir, options: 'czf')
+      def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
       end
 
-      def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:)
-        cmd = %W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)
-        _output, status = Gitlab::Popen.popen(cmd)
-        status.zero?
-      end
+      private
 
       def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:)
         cmd = %W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})
@@ -30,13 +26,14 @@ module Gitlab
       end
 
       def tar_with_options(archive:, dir:, options:)
-        cmd = %W(tar -#{options} #{archive} -C #{dir} .)
-        _output, status = Gitlab::Popen.popen(cmd)
-        status.zero?
+        execute(%W(tar -#{options} #{archive} -C #{dir} .))
       end
 
       def untar_with_options(archive:, dir:, options:)
-        cmd = %W(tar -#{options} #{archive} -C #{dir})
+        execute(%W(tar -#{options} #{archive} -C #{dir}))
+      end
+
+      def execute(cmd)
         _output, status = Gitlab::Popen.popen(cmd)
         status.zero?
       end

lib/gitlab/import_export/import_export.yml

 # Model relationships to be included in the project import/export
-:project_tree:
-  - :issues:
+project_tree:
+  - issues:
     - :notes
   - :labels
   - :milestones
   - :snippets
   - :releases
   - :events
-  - :project_members:
+  - project_members:
     - :user
-  - :merge_requests:
+  - merge_requests:
     - :merge_request_diff
     - :notes
-  - :ci_commits:
+  - ci_commits:
     - :statuses
 
 # Only include the following attributes for the models specified.
-:attributes_only:
-  :project:
+included_attributes:
+  project:
     - :name
     - :path
     - :description
@@ -28,12 +28,12 @@
     - :snippets_enabled
     - :visibility_level
     - :archived
-  :user:
+  user:
     - :id
     - :email
     - :username
 
 # Do not include the following attributes for the models specified.
-:attributes_except:
-  :snippets:
+excluded_attributes:
+  snippets:
     - :expired_at
\ No newline at end of file

lib/gitlab/import_export/import_export_reader.rb

@@ -4,7 +4,7 @@ module Gitlab
       extend self
 
       def project_tree
-        { only: atts_only[:project], include: build_hash(tree) }
+        { only: included_attributes[:project], include: build_hash(tree) }
       end
 
       def tree
@@ -14,24 +14,24 @@ module Gitlab
       private
 
       def config
-        @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml')
+        @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access
       end
 
-      def atts_only
-        config[:attributes_only]
+      def included_attributes
+        config[:included_attributes] || {}
       end
 
-      def atts_except
-        config[:attributes_except]
+      def excluded_attributes
+        config[:excluded_attributes] || {}
       end
 
       def build_hash(array)
-        array.map do |el|
-          if el.is_a?(Hash)
-            process_include(el)
+        array.map do |model_object|
+          if model_object.is_a?(Hash)
+            process_include(model_object)
           else
-            only_except_hash = check_only_and_except(el)
-            only_except_hash.empty? ? el : { el => only_except_hash }
+            only_except_hash = check_only_and_except(model_object)
+            only_except_hash.empty? ? model_object : { model_object => only_except_hash }
           end
         end
       end
@@ -82,12 +82,12 @@ module Gitlab
 
       def check_only(value)
         key = key_from_hash(value)
-        atts_only[key].nil? ? {} : { only: atts_only[key] }
+        included_attributes[key].nil? ? {} : { only: included_attributes[key] }
       end
 
       def check_except(value)
         key = key_from_hash(value)
-        atts_except[key].nil? ? {} : { except: atts_except[key] }
+        excluded_attributes[key].nil? ? {} : { except: excluded_attributes[key] }
       end
 
       def key_from_hash(value)

lib/gitlab/import_export/project_tree_restorer.rb

@@ -14,7 +14,8 @@ module Gitlab
         @tree_hash = ActiveSupport::JSON.decode(json)
         @project_members = @tree_hash.delete('project_members')
         create_relations
-      rescue
+      rescue => e
+        # TODO: handle errors better, move them to a shared thing
         false
       end
 
@@ -83,7 +84,7 @@ module Gitlab
 
       def relation_from_factory(relation, relation_hash)
         Gitlab::ImportExport::RelationFactory.create(
-          relation_sym: relation, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map)
+          relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map)
       end
     end
   end

lib/gitlab/import_export/project_tree_saver.rb

@@ -3,19 +3,13 @@ module Gitlab
     class ProjectTreeSaver
       attr_reader :full_path
 
-      def initialize(project: , shared: )
+      def initialize(project:, shared:)
         @project = project
         @export_path = shared.export_path
-      end
-
-      def save
         @full_path = File.join(@export_path, project_filename)
-        save_to_disk
       end
 
-      private
-
-      def save_to_disk
+      def save
         FileUtils.mkdir_p(@export_path)
         File.write(full_path, project_json_tree)
         true
@@ -24,6 +18,8 @@ module Gitlab
         false
       end
 
+      private
+
       # TODO remove magic keyword and move it to a shared config
       def project_filename
         "project.json"

spec/lib/gitlab/import_export/import_export_reader_spec.rb

-require 'rspec'
+require 'spec_helper'
 
 describe Gitlab::ImportExport::ImportExportReader do
 
@@ -17,7 +17,7 @@ describe Gitlab::ImportExport::ImportExportReader do
   end
 
   it 'should generate hash from project tree config' do
-    allow(described_class).to receive(:config).and_return(YAML.load_file(test_config))
+    allow(described_class).to receive(:config).and_return(YAML.load_file(test_config).deep_symbolize_keys)
 
     expect(described_class.project_tree).to eq(project_tree_hash)
   end

spec/lib/gitlab/import_export/members_mapper_spec.rb

 require 'spec_helper'
 
 describe Gitlab::ImportExport::MembersMapper, services: true do
-  describe :map do
+  describe 'map members' do
 
     let(:user) { create(:user) }
     let(:project) { create(:project, :public, name: 'searchable_project') }
@@ -31,7 +31,7 @@ describe Gitlab::ImportExport::MembersMapper, services: true do
     end
 
     let(:members_mapper) do
-      Gitlab::ImportExport::MembersMapper.new(
+      described_class.new(
         exported_members: exported_members, user: user, project_id: project.id)
     end
 

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

 require 'spec_helper'
 
 describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
-  describe :restore do
+  describe 'restore project tree' do
 
     let(:user) { create(:user) }
     let(:namespace) { create(:namespace, owner: user) }
-    let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(path: "lib/gitlab/import_export/", user: user, project_path: 'project', namespace_id: namespace.id) }
+    let(:project_tree_restorer) { described_class.new(path: Rails.root.join("spec/lib/gitlab/import_export/"), user: user, project_path: 'project', namespace_id: namespace.id) }
 
     context 'JSON' do
       let(:restored_project_json) do

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

 require 'spec_helper'
 
 describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
-  describe :save do
+  describe 'saves the project tree into a json object' do
 
-    # TODO refactor this into a setup method
-
-    let(:user) { create(:user) }
-    let(:issue) { create(:issue, assignee: user) }
-    let(:merge_request) { create(:merge_request) }
-    let(:label) { create(:label) }
-    let(:snippet) { create(:project_snippet) }
-    let(:commit_status) { create(:commit_status) }
-    let(:release) { create(:release) }
-    let!(:project) do
-      create(:project,
-             :public,
-             name: 'searchable_project',
-             issues: [issue],
-             merge_requests: [merge_request],
-             labels: [label],
-             snippets: [snippet],
-             releases: [release]
-            )
-    end
-    let!(:ci_commit) { create(:ci_commit, project: project, sha: merge_request.last_commit.id, ref: merge_request.source_branch, statuses: [commit_status]) }
-    let!(:milestone) { create(:milestone, title: "Milestone v1.2", project: project) }
-    let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
     let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
-    let(:project_tree_saver) { Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: shared) }
-    let!(:issue_note) { create(:note, note: ":+1: issue", noteable: issue) }
-    let!(:merge_request_note) { create(:note, note: ":+1: merge_request", noteable: merge_request) }
+    let(:project_tree_saver) { described_class.new(project: project, shared: shared) }
+    let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
+    let(:user) { create(:user) }
+    let(:project) { setup_project }
 
-    before(:each) do
+    before do
       project.team << [user, :master]
       allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
     end
 
-    after(:each) do
+    after do
       FileUtils.rm_rf(export_path)
     end
 
@@ -109,6 +87,30 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
     end
   end
 
+  def setup_project
+    issue = create(:issue, assignee: user)
+    merge_request = create(:merge_request)
+    label = create(:label)
+    snippet = create(:project_snippet)
+    commit_status = create(:commit_status)
+    release = create(:release)
+
+    project = create(:project,
+                     :public,
+                     issues: [issue],
+                     merge_requests: [merge_request],
+                     labels: [label],
+                     snippets: [snippet],
+                     releases: [release]
+                    )
+
+    create(:ci_commit, project: project, sha: merge_request.last_commit.id, ref: merge_request.source_branch, statuses: [commit_status])
+    create(:milestone, project: project)
+    create(:note, noteable: issue)
+    create(:note, noteable: merge_request)
+    project
+  end
+
   def project_json(filename)
     JSON.parse(IO.read(filename))
   end

spec/lib/gitlab/import_export/repo_bundler_spec.rb

 require 'spec_helper'
 
 describe Gitlab::ImportExport::RepoBundler, services: true do
-  describe :bundle do
+  describe 'bundle a project Git repo' do
 
     let(:user) { create(:user) }
     let!(:project) { create(:project, :public, name: 'searchable_project') }
     let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
     let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
-    let(:bundler) { Gitlab::ImportExport::RepoBundler.new(project: project, shared: shared) }
+    let(:bundler) { described_class.new(project: project, shared: shared) }
 
-    before(:each) do
+    before do
       project.team << [user, :master]
       allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
     end
 
-    after(:each) do
+    after do
       FileUtils.rm_rf(export_path)
     end
 

spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb

 require 'spec_helper'
 
 describe Gitlab::ImportExport::WikiRepoBundler, services: true do
-  describe :bundle do
+  describe 'bundle a wiki Git repo' do
 
     let(:user) { create(:user) }
     let!(:project) { create(:project, :public, name: 'searchable_project') }
     let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
     let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
-    let(:wiki_bundler) { Gitlab::ImportExport::WikiRepoBundler.new(project: project, shared: shared) }
+    let(:wiki_bundler) { described_class.new(project: project, shared: shared) }
     let!(:project_wiki) { ProjectWiki.new(project, user) }
 
-    before(:each) do
+    before do
       project.team << [user, :master]
       allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
       project_wiki.wiki
       project_wiki.create_page("index", "test content")
     end
 
-    after(:each) do
+    after do
       FileUtils.rm_rf(export_path)
     end
 

spec/support/import_export/import_export.yml

 # Class relationships to be included in the project import/export
-:project_tree:
+project_tree:
   - :issues
   - :labels
-  - :merge_requests:
+  - merge_requests:
     - :merge_request_diff
     - :merge_request_test
-  - :commit_statuses:
+  - commit_statuses:
     - :commit
 
-:attributes_only:
-  :project:
+included_attributes:
+  project:
     - :name
     - :path
-  :merge_requests:
+  merge_requests:
     - :id
 
-:attributes_except:
-  :merge_requests:
+excluded_attributes:
+  merge_requests:
     - :iid
\ No newline at end of file