Merge branch '205383-canonical-migration' into 'master'

Populate canonical emails

Closes #205383

See merge request gitlab-org/gitlab!28298

db/migrate/20200312053852_populate_canonical_emails.rb

+# frozen_string_literal: true
+
+class PopulateCanonicalEmails < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'users'
+
+    scope :with_gmail, -> { select(:id, :email).where("email ILIKE '%gmail.com'") }
+  end
+
+  # Limited to *@gmail.com addresses only as a first iteration, because we know
+  # Gmail ignores `.` appearing in the Agent name, as well as anything after `+`
+
+  def up
+    # batch size is the default, 1000
+    migration = Gitlab::BackgroundMigration::PopulateCanonicalEmails
+    migration_name = migration.to_s.demodulize
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      User.with_gmail,
+      migration_name,
+      1.minute)
+  end
+
+  def down
+    # no-op
+  end
+end

db/structure.sql

@@ -13007,6 +13007,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200311165635
 20200311192351
 20200311214912
+20200312053852
 20200312125121
 20200312160532
 20200312163407

lib/gitlab/background_migration/populate_canonical_emails.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class to populate new rows of UserCanonicalEmail based on existing email addresses
+    class PopulateCanonicalEmails
+      def perform(start_id, stop_id)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO
+            user_canonical_emails (
+                user_id,
+                canonical_email,
+                created_at,
+                updated_at
+            )
+            SELECT users.id AS user_id,
+                   concat(translate(split_part(split_part(users.email, '@', 1), '+', 1), '.', ''), '@gmail.com') AS canonical_email,
+                   NOW() AS created_at,
+                   NOW() AS updated_at
+             FROM users
+            WHERE users.email ILIKE '%@gmail.com'
+              AND users.id BETWEEN #{start_id} AND #{stop_id}
+            ON CONFLICT DO NOTHING;
+        SQL
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/populate_canonical_emails_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::PopulateCanonicalEmails, :migration, schema: 20200312053852 do
+  let(:migration) { described_class.new }
+
+  let_it_be(:users_table) { table(:users) }
+  let_it_be(:user_canonical_emails_table) { table(:user_canonical_emails) }
+
+  let_it_be(:users) { users_table.all }
+  let_it_be(:user_canonical_emails) { user_canonical_emails_table.all }
+
+  subject { migration.perform(1, 1) }
+
+  describe 'gmail users' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:original_email, :expected_result) do
+      'legitimateuser@gmail.com'                            | 'legitimateuser@gmail.com'
+      'userwithplus+somestuff@gmail.com'                    | 'userwithplus@gmail.com'
+      'user.with.periods@gmail.com'                         | 'userwithperiods@gmail.com'
+      'user.with.periods.and.plus+someotherstuff@gmail.com' | 'userwithperiodsandplus@gmail.com'
+    end
+
+    with_them do
+      it 'generates the correct canonical email' do
+        create_user(email: original_email, id: 1)
+
+        subject
+
+        result = canonical_emails
+        expect(result.count).to eq 1
+        expect(result.first).to match({
+          'user_id' => 1,
+          'canonical_email' => expected_result
+        })
+      end
+    end
+  end
+
+  describe 'non gmail.com domain users' do
+    %w[
+      legitimateuser@somedomain.com
+      userwithplus+somestuff@other.com
+      user.with.periods@gmail.org
+      user.with.periods.and.plus+someotherstuff@orangmail.com
+    ].each do |non_gmail_address|
+      it 'does not generate a canonical email' do
+        create_user(email: non_gmail_address, id: 1)
+
+        subject
+
+        expect(canonical_emails(user_id: 1).count).to eq 0
+      end
+    end
+  end
+
+  describe 'gracefully handles missing records' do
+    specify { expect { subject }.not_to raise_error }
+  end
+
+  describe 'gracefully handles existing records, some of which may have an already-existing identical canonical_email field' do
+    let_it_be(:user_one) { create_user(email: "example.user@gmail.com", id: 1) }
+    let_it_be(:user_two) { create_user(email: "exampleuser@gmail.com", id: 2) }
+    let_it_be(:user_email_one) { user_canonical_emails.create(canonical_email: "exampleuser@gmail.com", user_id: user_one.id) }
+
+    subject { migration.perform(1, 2) }
+
+    it 'only creates one record' do
+      subject
+
+      expect(canonical_emails.count).not_to be_nil
+    end
+  end
+
+  def create_user(attributes)
+    default_attributes = {
+        projects_limit: 0
+    }
+
+    users.create(default_attributes.merge!(attributes))
+  end
+
+  def canonical_emails(user_id: nil)
+    filter_by_id = user_id ? "WHERE user_id = #{user_id}" : ""
+
+    ApplicationRecord.connection.execute <<~SQL
+      SELECT canonical_email, user_id
+      FROM user_canonical_emails
+      #{filter_by_id};
+    SQL
+  end
+end