Import directory tree created with hashed storage using import rake task

lib/gitlab/bare_repository_import/importer.rb

@@ -14,13 +14,13 @@ module Gitlab
         repos_to_import.each do |repo_path|
           bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path)
 
-          if bare_repo.hashed? || bare_repo.wiki?
+          unless bare_repo.processable?
             log " * Skipping repo #{bare_repo.repo_path}".color(:yellow)
 
             next
           end
 
-          log "Processing #{repo_path}".color(:yellow)
+          log "Processing #{repo_path} -> #{bare_repo.project_full_path}".color(:yellow)
 
           new(user, bare_repo).create_project_if_needed
         end
@@ -62,6 +62,11 @@ module Gitlab
         if project.persisted? && mv_repo(project)
           log " * Created #{project.name} (#{project_full_path})".color(:green)
 
+          # We'd need to keep track of project full path otherwise directory tree
+          # created with hashed storage enabled cannot be usefully imported using
+          # the import rake task.
+          project.write_repository_config(:fullpath, project.full_path)
+
           ProjectCacheWorker.perform_async(project.id)
         else
           log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red)

lib/gitlab/bare_repository_import/repository.rb

@@ -6,39 +6,55 @@ module Gitlab
       def initialize(root_path, repo_path)
         @root_path = root_path
         @repo_path = repo_path
-
         @root_path << '/' unless root_path.ends_with?('/')
 
+        full_path =
+          if hashed? && !wiki?
+            repository.config.get('gitlab.fullpath')
+          else
+            repo_relative_path
+          end
+
         # Split path into 'all/the/namespaces' and 'project_name'
-        @group_path, _, @project_name = repo_relative_path.rpartition('/')
+        @group_path, _, @project_name = full_path.to_s.rpartition('/')
       end
 
       def wiki_exists?
         File.exist?(wiki_path)
       end
 
-      def wiki?
-        @wiki ||= repo_path.end_with?('.wiki.git')
-      end
-
       def wiki_path
         @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git')
       end
 
-      def hashed?
-        @hashed ||= group_path.start_with?('@hashed')
-      end
-
       def project_full_path
         @project_full_path ||= "#{group_path}/#{project_name}"
       end
 
+      def processable?
+        return false if wiki?
+
+        group_path.present? && project_name.present?
+      end
+
       private
 
+      def wiki?
+        @wiki ||= repo_path.end_with?('.wiki.git')
+      end
+
+      def hashed?
+        @hashed ||= repo_relative_path.include?('@hashed')
+      end
+
       def repo_relative_path
         # Remove root path and `.git` at the end
         repo_path[@root_path.size...-4]
       end
+
+      def repository
+        @repository ||= Rugged::Repository.new(repo_path)
+      end
     end
   end
 end

spec/lib/gitlab/bare_repository_import/importer_spec.rb

 require 'spec_helper'
 
 describe Gitlab::BareRepositoryImport::Importer, repository: true do
+  let(:gitlab_shell) { Gitlab::Shell.new }
   let!(:admin) { create(:admin) }
   let!(:base_dir) { Dir.mktmpdir + '/' }
   let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) }
@@ -75,7 +76,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
       end
 
       it 'creates the Git repo in disk' do
-        FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
+        create_bare_repository("#{project_path}.git")
 
         importer.create_project_if_needed
 
@@ -130,7 +131,7 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
     end
 
     it 'creates the Git repo in disk' do
-      FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
+      create_bare_repository("#{project_path}.git")
 
       importer.create_project_if_needed
 
@@ -165,8 +166,8 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
     it_behaves_like 'importing a repository'
 
     it 'creates the Wiki git repo in disk' do
-      FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git"))
-      FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git"))
+      create_bare_repository("#{project_path}.git")
+      create_bare_repository("#{project_path}.wiki.git")
 
       expect(Projects::CreateService).to receive(:new).with(admin, hash_including(skip_wiki: true,
                                                                                   import_type: 'bare_repository')).and_call_original
@@ -192,4 +193,9 @@ describe Gitlab::BareRepositoryImport::Importer, repository: true do
       end
     end
   end
+
+  def create_bare_repository(project_path)
+    repo_path = File.join(base_dir, project_path)
+    Gitlab::Git::Repository.create(repo_path, bare: true)
+  end
 end

spec/lib/gitlab/bare_repository_import/repository_spec.rb

 require 'spec_helper'
 
 describe ::Gitlab::BareRepositoryImport::Repository do
-  let(:project_repo_path) { described_class.new('/full/path/', '/full/path/to/repo.git') }
+  context 'legacy storage' do
+    subject { described_class.new('/full/path/', '/full/path/to/repo.git') }
 
-  it 'stores the repo path' do
-    expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git')
-  end
+    it 'stores the repo path' do
+      expect(subject.repo_path).to eq('/full/path/to/repo.git')
+    end
 
-  it 'stores the group path' do
-    expect(project_repo_path.group_path).to eq('to')
-  end
+    it 'stores the group path' do
+      expect(subject.group_path).to eq('to')
+    end
 
-  it 'stores the project name' do
-    expect(project_repo_path.project_name).to eq('repo')
-  end
+    it 'stores the project name' do
+      expect(subject.project_name).to eq('repo')
+    end
 
-  it 'stores the wiki path' do
-    expect(project_repo_path.wiki_path).to eq('/full/path/to/repo.wiki.git')
-  end
+    it 'stores the wiki path' do
+      expect(subject.wiki_path).to eq('/full/path/to/repo.wiki.git')
+    end
+
+    describe '#processable?' do
+      it 'returns false if it is a wiki' do
+        subject = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git')
 
-  describe '#wiki?' do
-    it 'returns true if it is a wiki' do
-      wiki_path = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git')
+        expect(subject.processable?).to eq(false)
+      end
 
-      expect(wiki_path.wiki?).to eq(true)
+      it 'returns true when group and project name are present' do
+        expect(subject.processable?).to eq(true)
+      end
     end
 
-    it 'returns false if it is not a wiki' do
-      expect(project_repo_path.wiki?).to eq(false)
+    describe '#project_full_path' do
+      it 'returns the project full path' do
+        expect(subject.repo_path).to eq('/full/path/to/repo.git')
+        expect(subject.project_full_path).to eq('to/repo')
+      end
+
+      it 'with no trailing slash in the root path' do
+        repo_path = described_class.new('/full/path', '/full/path/to/repo.git')
+
+        expect(repo_path.project_full_path).to eq('to/repo')
+      end
     end
   end
 
-  describe '#hashed?' do
-    it 'returns true if it is a hashed folder' do
-      path = described_class.new('/full/path/', '/full/path/@hashed/my.repo.git')
+  context 'hashed storage' do
+    let(:gitlab_shell) { Gitlab::Shell.new }
+    let(:repository_storage) { 'default' }
+    let(:root_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
+    let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
+    let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" }
+    let(:repo_path) { File.join(root_path, "#{hashed_path}.git") }
+    let(:wiki_path) { File.join(root_path, "#{hashed_path}.wiki.git") }
 
-      expect(path.hashed?).to eq(true)
+    before do
+      gitlab_shell.add_repository(repository_storage, hashed_path)
+      repository = Rugged::Repository.new(repo_path)
+      repository.config['gitlab.fullpath'] = 'to/repo'
     end
 
-    it 'returns false if it is not a hashed folder' do
-      expect(project_repo_path.hashed?).to eq(false)
+    after do
+      gitlab_shell.remove_repository(root_path, hashed_path)
     end
-  end
 
-  describe '#project_full_path' do
-    it 'returns the project full path' do
-      expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git')
-      expect(project_repo_path.project_full_path).to eq('to/repo')
+    subject { described_class.new(root_path, repo_path) }
+
+    it 'stores the repo path' do
+      expect(subject.repo_path).to eq(repo_path)
+    end
+
+    it 'stores the wiki path' do
+      expect(subject.wiki_path).to eq(wiki_path)
+    end
+
+    it 'reads the group path from .git/config' do
+      expect(subject.group_path).to eq('to')
+    end
+
+    it 'reads the project name from .git/config' do
+      expect(subject.project_name).to eq('repo')
     end
 
-    it 'with no trailing slash in the root path' do
-      repo_path = described_class.new('/full/path', '/full/path/to/repo.git')
+    describe '#processable?' do
+      it 'returns false if it is a wiki' do
+        subject = described_class.new(root_path, wiki_path)
+
+        expect(subject.processable?).to eq(false)
+      end
+
+      it 'returns false when group and project name are missing' do
+        repository = Rugged::Repository.new(repo_path)
+        repository.config.delete('gitlab.fullpath')
+
+        expect(subject.processable?).to eq(false)
+      end
+
+      it 'returns true when group and project name are present' do
+        expect(subject.processable?).to eq(true)
+      end
+    end
 
-      expect(repo_path.project_full_path).to eq('to/repo')
+    describe '#project_full_path' do
+      it 'returns the project full path' do
+        expect(subject.project_full_path).to eq('to/repo')
+      end
     end
   end
 end