Fix issue import to accept export format

We want to be able to export issues and then use that export file to
import the issues to another GitLab project.

Prior to this change, the format of the export file did not match the
structure of the file that the issue importer expects. This change
expands the importer to first look for the know export column names and
then fallback to positional values if they are not present.

Addresses:
- https://gitlab.com/gitlab-org/gitlab/issues/199038
- https://gitlab.com/gitlab-org/gitlab/issues/30931

app/services/issues/import_csv_service.rb

@@ -21,8 +21,19 @@ module Issues
     def process_csv
       csv_data = @csv_io.open(&:read).force_encoding(Encoding::UTF_8)
 
-      CSV.new(csv_data, col_sep: detect_col_sep(csv_data.lines.first), headers: true).each.with_index(2) do |row, line_no|
-        issue = Issues::CreateService.new(@project, @user, title: row[0], description: row[1]).execute
+      csv_parsing_params = {
+        col_sep: detect_col_sep(csv_data.lines.first),
+        headers: true,
+        header_converters: :symbol
+      }
+
+      CSV.new(csv_data, csv_parsing_params).each.with_index(2) do |row, line_no|
+        issue_attributes = {
+          title:       row[:title],
+          description: row[:description]
+        }
+
+        issue = Issues::CreateService.new(@project, @user, issue_attributes).execute
 
         if issue.persisted?
           @results[:success] += 1

changelogs/unreleased/jhyson-issue-import-export-consistency.yml

+---
+title: Fix issue importer so it matches issue export format
+merge_request: 25896
+author:
+type: fixed

doc/user/project/issues/csv_import.md

@@ -3,7 +3,7 @@
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23532) in GitLab 11.7.
 
 Issues can be imported to a project by uploading a CSV file with the columns
-`title` and `description`, in that order.
+`title` and `description`.
 
 The user uploading the CSV file will be set as the author of the imported issues.
 
@@ -31,9 +31,9 @@ to you once the import is complete.
 
 When importing issues from a CSV file, it must be formatted in a certain way:
 
-- **header row:** CSV files must contain a header row where the first column header
-  is `title` and the second is `description`. If additional columns are present, they
-  will be ignored.
+- **header row:** CSV files must include the following headers:
+`title` and `description`. The case of the headers does not matter.
+- **columns:** Data from columns beyond `title` and `description` are not imported.
 - **separators:** The column separator is automatically detected from the header row.
   Supported separator characters are: commas (`,`), semicolons (`;`), and tabs (`\t`).
   The row separator can be either `CRLF` or `LF`.

ee/spec/services/issues/export_csv_service_spec.rb

@@ -54,6 +54,10 @@ describe Issues::ExportCsvService do
                     time_estimate: 72000)
     end
 
+    it 'includes the columns required for import' do
+      expect(csv.headers).to include('Title', 'Description')
+    end
+
     specify 'iid' do
       expect(csv[0]['Issue ID']).to eq issue.iid.to_s
     end

spec/fixtures/csv_gitlab_export.csv

+Issue ID,URL,Title,State,Description,Author,Author Username,Assignee,Assignee Username,Confidential,Locked,Due Date,Created At (UTC),Updated At (UTC),Closed At (UTC),Milestone,Weight,Labels,Time Estimate,Time Spent,Epic ID,Epic Title
+1,http://localhost:3000/jashkenas/underscore/issues/1,Title,Open,,Elva Jerde,jamel,Tierra Effertz,aurora_hahn,No,No,,2020-01-17 10:36:26,2020-02-19 10:36:26,,v1.0,,"Brene,Cutlass,Escort,GVM",0,0,,
+3,http://localhost:3000/jashkenas/underscore/issues/3,Nihil impedit neque quos totam ut aut enim cupiditate doloribus molestiae.,Open,Omnis aliquid sint laudantium quam.,Marybeth Goodwin,rocio.blanda,Annemarie Von,reynalda_howe,No,No,,2020-01-23 10:36:26,2020-02-19 10:36:27,,v1.0,,"Brene,Cutlass,Escort,GVM",0,0,,
+34,http://localhost:3000/jashkenas/underscore/issues/34,Dismiss Cipher with no integrity,Open,,Marybeth Goodwin,rocio.blanda,"","",No,No,,2020-02-19 10:38:49,2020-02-19 10:38:49,,,,,0,0,,
+35,http://localhost:3000/jashkenas/underscore/issues/35,Test Title,Open,Test Description,Marybeth Goodwin,rocio.blanda,"","",No,No,,2020-02-19 10:38:49,2020-02-19 10:38:49,,,,,0,0,,

spec/services/issues/import_csv_service_spec.rb

@@ -27,6 +27,29 @@ describe Issues::ImportCsvService do
       end
     end
 
+    context 'with a file generated by Gitlab CSV export' do
+      let(:file) { fixture_file_upload('spec/fixtures/csv_gitlab_export.csv') }
+
+      it 'imports the CSV without errors' do
+        expect_next_instance_of(Notify) do |instance|
+          expect(instance).to receive(:import_issues_csv_email)
+        end
+
+        expect(subject[:success]).to eq(4)
+        expect(subject[:error_lines]).to eq([])
+        expect(subject[:parse_error]).to eq(false)
+      end
+
+      it 'correctly sets the issue attributes' do
+        expect { subject }.to change { project.issues.count }.by 4
+
+        expect(project.issues.reload.last).to have_attributes(
+          title: 'Test Title',
+          description: 'Test Description'
+        )
+      end
+    end
+
     context 'comma delimited file' do
       let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') }
 
@@ -39,6 +62,15 @@ describe Issues::ImportCsvService do
         expect(subject[:error_lines]).to eq([])
         expect(subject[:parse_error]).to eq(false)
       end
+
+      it 'correctly sets the issue attributes' do
+        expect { subject }.to change { project.issues.count }.by 3
+
+        expect(project.issues.reload.last).to have_attributes(
+          title: 'Title with quote"',
+          description: 'Description'
+        )
+      end
     end
 
     context 'tab delimited file with error row' do
@@ -53,6 +85,15 @@ describe Issues::ImportCsvService do
         expect(subject[:error_lines]).to eq([3])
         expect(subject[:parse_error]).to eq(false)
       end
+
+      it 'correctly sets the issue attributes' do
+        expect { subject }.to change { project.issues.count }.by 2
+
+        expect(project.issues.reload.last).to have_attributes(
+          title: 'Hello',
+          description: 'World'
+        )
+      end
     end
 
     context 'semicolon delimited file with CRLF' do
@@ -67,6 +108,15 @@ describe Issues::ImportCsvService do
         expect(subject[:error_lines]).to eq([4])
         expect(subject[:parse_error]).to eq(false)
       end
+
+      it 'correctly sets the issue attributes' do
+        expect { subject }.to change { project.issues.count }.by 3
+
+        expect(project.issues.reload.last).to have_attributes(
+          title: 'Hello',
+          description: 'World'
+        )
+      end
     end
   end
 end