Automatic merge of gitlab-org/gitlab-ce master

lib/gitlab/project_template.rb

@@ -24,6 +24,14 @@ module Gitlab
       "#{preview}.git"
     end
 
+    def project_path
+      URI.parse(preview).path.sub(%r{\A/}, '')
+    end
+
+    def uri_encoded_project_path
+      ERB::Util.url_encode(project_path)
+    end
+
     def ==(other)
       name == other.name && title == other.title
     end

lib/tasks/gitlab/update_templates.rake

@@ -40,7 +40,6 @@ namespace :gitlab do
 
     templates.each do |template|
       params = {
-        import_url: template.clone_url,
         namespace_id: tmp_namespace.id,
         path: template.name,
         skip_wiki: true
@@ -53,22 +52,46 @@ namespace :gitlab do
         raise "Failed to create project: #{project.errors.messages}"
       end
 
-      loop do
-        if project.import_finished?
-          puts "Import finished for #{template.name}"
-          break
+      uri_encoded_project_path = template.uri_encoded_project_path
+
+      # extract a concrete commit for signing off what we actually downloaded
+      # this way we do the right thing even if the repository gets updated in the meantime
+      get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits",
+        query: { page: 1, per_page: 1 }
+      )
+      raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success?
+
+      commit_sha = get_commits_response.parsed_response.dig(0, 'id')
+
+      project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}"
+      commit_message = <<~MSG
+        Initialized from '#{template.title}' project template
+
+        Template repository: #{template.preview}
+        Commit SHA: #{commit_sha}
+      MSG
+
+      Dir.mktmpdir do |tmpdir|
+        Dir.chdir(tmpdir) do
+          Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz'])
+          Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz'])
+          extracted_project_basename = Dir['*/'].first
+          Dir.chdir(extracted_project_basename) do
+            Gitlab::TaskHelpers.run_command!(%w(git init))
+            Gitlab::TaskHelpers.run_command!(%w(git add .))
+            Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message])
+
+            # Hacky workaround to push to the project in a way that works with both GDK and the test environment
+            Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+              Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"])
+            end
+            Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master'])
+          end
         end
-
-        if project.import_failed?
-          raise "Failed to import from #{project_params[:import_url]}"
-        end
-
-        puts "Waiting for the import to finish"
-
-        sleep(5)
-        project.reset
       end
 
+      project.reset
+
       Projects::ImportExport::ExportService.new(project, admin).execute
       downloader.call(project.export_file, template.archive_path)
 

spec/lib/gitlab/project_template_spec.rb

@@ -28,6 +28,18 @@ describe Gitlab::ProjectTemplate do
     end
   end
 
+  describe '#project_path' do
+    subject { described_class.new('name', 'title', 'description', 'https://gitlab.com/some/project/path').project_path }
+
+    it { is_expected.to eq 'some/project/path' }
+  end
+
+  describe '#uri_encoded_project_path' do
+    subject { described_class.new('name', 'title', 'description', 'https://gitlab.com/some/project/path').uri_encoded_project_path }
+
+    it { is_expected.to eq 'some%2Fproject%2Fpath' }
+  end
+
   describe '.find' do
     subject { described_class.find(query) }
 

spec/requests/api/discussions_spec.rb

@@ -17,6 +17,8 @@ describe API::Discussions do
     let!(:note) { create(:system_note, noteable: merge_request, project: project, note: cross_reference) }
     let!(:note_metadata) { create(:system_note_metadata, note: note, action: 'cross_reference') }
     let(:cross_reference) { "test commit #{commit.to_reference(project)}" }
+    let(:pat) { create(:personal_access_token, user: user) }
+
     let(:url) { "/projects/#{project.id}/merge_requests/#{merge_request.iid}/discussions" }
 
     before do
@@ -30,7 +32,7 @@ describe API::Discussions do
       new_note = create(:system_note, noteable: merge_request, project: project, note: new_cross_reference)
       create(:system_note_metadata, note: new_note, action: 'cross_reference')
 
-      get api(url, user)
+      get api(url, user, personal_access_token: pat)
       expect(response).to have_gitlab_http_status(200)
       expect(json_response.count).to eq(1)
       expect(json_response.first['notes'].count).to eq(1)
@@ -45,7 +47,7 @@ describe API::Discussions do
       expect_any_instance_of(Repository).not_to receive(:find_commit).with(commit.id)
 
       control = ActiveRecord::QueryRecorder.new do
-        get api(url, user)
+        get api(url, user, personal_access_token: pat)
       end
 
       expect(response).to have_gitlab_http_status(200)
@@ -57,7 +59,7 @@ describe API::Discussions do
 
       RequestStore.clear!
 
-      expect { get api(url, user) }.not_to exceed_query_limit(control)
+      expect { get api(url, user, personal_access_token: pat) }.not_to exceed_query_limit(control)
       expect(response).to have_gitlab_http_status(200)
     end
   end

spec/tasks/gitlab/update_templates_rake_spec.rb

@@ -8,9 +8,18 @@ describe 'gitlab:update_project_templates rake task' do
   before do
     Rake.application.rake_require 'tasks/gitlab/update_templates'
     create(:admin)
+
     allow(Gitlab::ProjectTemplate)
       .to receive(:archive_directory)
         .and_return(Pathname.new(tmpdir))
+
+    # Gitlab::HTTP resolves the domain to an IP prior to WebMock taking effect, hence the wildcard
+    stub_request(:get, %r{^https://.*/api/v4/projects/gitlab-org%2Fproject-templates%2Frails/repository/commits\?page=1&per_page=1})
+      .to_return(
+        status: 200,
+        body: [{ id: '67812735b83cb42710f22dc98d73d42c8bf4d907' }].to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      )
   end
 
   after do