Merge branch '5960-consider-EE-ahead-of-CE' into 'master'

Update CE branch whenever EE is ahead of CE branch

Closes #5960

See merge request gitlab-org/gitlab-ee!5719

scripts/ee-specific-lines-check

@@ -7,8 +7,8 @@ git_version
 
 base = find_compare_base
 
-current_numstat = updated_diff_numstat(base.ce_merge_base, base.ee_fetch_base)
-updated_numstat = updated_diff_numstat(base.ce_updated_base, 'HEAD')
+current_numstat = updated_diff_numstat(base.ce_base, base.ee_base)
+updated_numstat = updated_diff_numstat(base.ce_head, base.ee_head)
 
 offenses = updated_numstat.select do |file, updated_delta|
   current_delta = current_numstat[file]
@@ -16,11 +16,11 @@ offenses = updated_numstat.select do |file, updated_delta|
   more_lines =
     current_delta &&
     updated_delta > current_delta &&
-    WHITELIST.all? { |pattern| !Dir.glob(pattern).include?(file) }
-
     # Don't complain if we're just adding 1 or 2 more lines, which could be
     # `prepend EE::Module` and a blank line that we want.
-  more_lines && !(current_delta == 0 && updated_delta <= 2)
+    !(current_delta == 0 && updated_delta <= 2)
+
+  more_lines && !WHITELIST.any? { |pattern| Dir.glob(pattern).include?(file) }
 end
 
 if offenses.empty?

scripts/ee_specific_check/ee_specific_check.rb

@@ -14,7 +14,8 @@ module EESpecificCheck
     'locale/gitlab.pot'
   ].freeze
 
-  CompareBase = Struct.new(:ce_merge_base, :ee_fetch_base, :ce_updated_base)
+  CompareBase = Struct.new(:ce_base, :ee_base, :ce_head, :ee_head)
+  GitStatus = Struct.new(:porcelain, :head)
 
   module_function
 
@@ -30,28 +31,24 @@ module EESpecificCheck
     git_clean
 
     ce_fetch_head = fetch_remote_ce_branch
+    ee_fetch_head = head_commit_sha
     ce_fetch_base = run_git_command("merge-base canonical-ce/master #{ce_fetch_head}")
     ce_merge_base = run_git_command("merge-base canonical-ce/master canonical-ee/master")
     ee_fetch_base = run_git_command("merge-base canonical-ee/master HEAD")
 
-    ce_updated_base =
-      if ce_fetch_head.start_with?('canonical-ce') || # No specific CE branch
-          ce_fetch_base == ce_merge_base # Up-to-date, no rebase needed
-        ce_merge_base
-      else
-        checkout_and_rebase_ce_fetch_head_onto_ce_merge_base(
-          ce_fetch_head, ce_fetch_base, ce_merge_base)
-      end
+    ce_updated_head =
+      find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
 
-    CompareBase.new(ce_merge_base, ee_fetch_base, ce_updated_base)
+    CompareBase.new(
+      ce_merge_base, ee_fetch_base, ce_updated_head, ee_fetch_head)
   end
 
   def setup_canonical_remotes
     run_git_command(
       "remote add canonical-ee https://gitlab.com/gitlab-org/gitlab-ee.git",
       "remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-ce.git",
-      "fetch canonical-ee master --quiet",
-      "fetch canonical-ce master --quiet")
+      "fetch canonical-ee master --quiet --depth=9999",
+      "fetch canonical-ce master --quiet --depth=9999")
   end
 
   def fetch_remote_ce_branch
@@ -59,19 +56,63 @@ module EESpecificCheck
 
     remote_to_fetch, branch_to_fetch = find_remote_ce_branch
 
-    run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet")
+    run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet --depth=9999")
 
     "#{remote_to_fetch}/#{branch_to_fetch}"
   end
 
-  def checkout_and_rebase_ce_fetch_head_onto_ce_merge_base(
-    ce_fetch_head, ce_fetch_base, ce_merge_base)
-    # So that we could switch back
-    head = head_commit_sha
+  def find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+    if git_ancestor?(ce_merge_base, ce_fetch_base)
+      say("CE is ahead of EE, finding backward CE head")
+      find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+    else
+      say("CE is behind of EE, finding forward CE head")
+      find_forward_ce_head(ce_merge_base, ce_fetch_head)
+    end
+  end
+
+  def git_ancestor?(ancestor, descendant)
+    run_git_command(
+      "merge-base --is-ancestor #{ancestor} #{descendant} && echo y") == 'y'
+  end
 
-    # Use detached HEAD so that we don't update HEAD
-    run_git_command("checkout -f #{ce_fetch_head}")
-    git_clean
+  def find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base)
+    if ce_fetch_head.start_with?('canonical-ce') || # No specific CE branch
+        ce_fetch_base == ce_merge_base # Up-to-date, no rebase needed
+      say("CE is up-to-date, using merge-base directly")
+      run_git_command("merge-base #{ce_merge_base} HEAD")
+    else
+      say("Performing rebase to remove commits in CE haven't merged into EE")
+      checkout_and_rebase(ce_merge_base, ce_fetch_base, ce_fetch_head)
+    end
+  end
+
+  def find_forward_ce_head(ce_merge_base, ce_fetch_head)
+    say("Performing merge with CE master for CE branch #{ce_fetch_head}")
+    with_detached_head(ce_fetch_head) do
+      run_git_command("merge #{ce_merge_base} -s recursive -X patience -m 'ee-specific-auto-merge'")
+
+      status = git_status
+
+      if status.porcelain == ''
+        status.head
+      else
+        run_git_command("merge --abort")
+
+        say <<~MESSAGE
+          💥 Git status not clean! This means there's a conflict in
+          💥 #{ce_fetch_head} with canonical-ce/master. Please resolve
+          💥 the conflict from CE master and retry this job.
+
+          ⚠️ Git status:
+
+          #{status.porcelain}
+        MESSAGE
+
+        exit(254)
+      end
+    end
+  end
 
   # We rebase onto the commit which is the latest commit presented in both
   # CE and EE, i.e. ce_merge_base, cutting off commits aren't merged into
@@ -109,13 +150,17 @@ module EESpecificCheck
   # where ce_merge_base..ee_fetch_base is the update-to-date
   # CE/EE difference and ce_fetch_head..HEAD is the changes we made in
   # CE and EE branches.
-    run_git_command("rebase --onto #{ce_merge_base} #{ce_fetch_base}~1 #{ce_fetch_head}")
+  def checkout_and_rebase(new_base, old_base, target_head)
+    with_detached_head(target_head) do
+      run_git_command("rebase --onto #{new_base} #{old_base} #{target_head}")
 
       status = git_status
 
-    if status == ''
-      head_commit_sha
+      if status.porcelain == ''
+        status.head
       else
+        run_git_command("rebase --abort")
+
         say <<~MESSAGE
           💥 Git status not clean! This shouldn't happen, but there are two
           💥 known issues. One can be worked around, and the other can't.
@@ -139,14 +184,24 @@ module EESpecificCheck
 
           ⚠️ Git status:
 
-        #{status}
+          #{status.porcelain}
         MESSAGE
 
-      run_git_command("rebase --abort")
-
         exit(255)
       end
+    end
+  end
+
+  def with_detached_head(target_head)
+    # So that we could switch back. CI sometimes doesn't have the branch,
+    # so we don't use current_branch here
+    head = current_head
 
+    # Use detached HEAD so that we don't update HEAD
+    run_git_command("checkout -f #{target_head}")
+    git_clean
+
+    yield
   ensure # ensure would still run if we call exit, don't worry
     # Make sure to switch back
     run_git_command("checkout -f #{head}")
@@ -158,7 +213,10 @@ module EESpecificCheck
   end
 
   def git_status
-    run_git_command("status --porcelain")
+    GitStatus.new(
+      run_git_command("status --porcelain"),
+      head_commit_sha
+    )
   end
 
   def git_clean
@@ -206,6 +264,10 @@ module EESpecificCheck
     @ce_repo_url ||= ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-ce.git').sub('gitlab-ee', 'gitlab-ce')
   end
 
+  def current_head
+    @current_head ||= ENV.fetch('CI_COMMIT_SHA', current_branch)
+  end
+
   def current_branch
     @current_branch ||= ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip
   end