Commit 2c8acf86 authored by Rémy Coutable's avatar Rémy Coutable

Merge branch '5682-ee-specific-check' into 'master'

Resolve "Create a new job to check that EE-specific lines are minimal"

Closes #5682

See merge request gitlab-org/gitlab-ee!5404
parents fb18e881 fd2ef33d
......@@ -459,6 +459,18 @@ ee-files-location-check:
- branches
- //@gitlab-org/gitlab-ee
ee-specific-lines-check:
<<: *dedicated-runner
stage: test
before_script: []
cache: {}
retry: 0
script:
- scripts/ee-specific-lines-check
only:
- branches
- //@gitlab-org/gitlab-ee
update-tests-metadata:
<<: *tests-metadata-state
<<: *only-canonical-masters
......
#!/usr/bin/env ruby
WHITELIST = [
'CHANGELOG-EE.md',
'config/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4946
'doc/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4948#note_59945483
'qa/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4997#note_59764702
'scripts/*',
'spec/javascripts/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/3871
'vendor/assets/javascripts/jasmine-jquery.js'
].freeze
require_relative 'ee_specific_check/ee_specific_check'
def run_git_command(cmd)
puts "=> Running `git #{cmd}`"
`git #{cmd}`
end
include EESpecificCheck # rubocop:disable Style/MixinUsage
run_git_command("remote add canonical-ee https://gitlab.com/gitlab-org/gitlab-ee.git")
run_git_command("remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-ce.git")
run_git_command("fetch canonical-ee master --quiet")
base = find_compare_base
new_files_in_this_branch_not_at_the_ee_top_level =
run_git_command("diff canonical-ee/master...HEAD --name-status --diff-filter=A -- ./ ':!ee' | cut -f2").lines.map(&:strip)
ce_repo_url = ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-ce.git').sub('gitlab-ee', 'gitlab-ce')
current_branch = ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip
minimal_ce_branch_name = current_branch.sub(/(\Aee\-|\-ee\z)/, '')
ls_remote_output = run_git_command("ls-remote #{ce_repo_url} \"*#{minimal_ce_branch_name}*\"")
remote_to_fetch = 'canonical-ce'
branch_to_fetch = 'master'
if ls_remote_output.include?(minimal_ce_branch_name)
remote_to_fetch = ce_repo_url
branch_to_fetch = ls_remote_output.scan(%r{(?<=refs/heads/).+}).sort_by(&:size).first
puts
puts "💪 We found the branch '#{branch_to_fetch}' in the #{ce_repo_url} repository. We will fetch it."
else
puts "⚠️ We did not find a branch that would match the current '#{current_branch}' branch in the #{ce_repo_url} repository. We will fetch 'master' instead."
puts "ℹ️ If you have a CE branch for the current branch, make sure that its name includes '#{minimal_ce_branch_name}'."
end
run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet")
run_git_command("diff #{base.ee_merge_base}...HEAD --name-status --diff-filter=A -- ./ ':!ee' | cut -f2").lines.map(&:strip)
ee_specific_files_in_ce_master_not_at_the_ee_top_level =
run_git_command("diff FETCH_HEAD..HEAD --name-status --diff-filter=A -- ./ ':!ee' | cut -f2").lines.map(&:strip)
run_git_command("diff #{base.ce_updated_base}..HEAD --name-status --diff-filter=A -- ./ ':!ee' | cut -f2").lines.map(&:strip)
new_ee_specific_files_not_at_the_ee_top_level =
new_files_in_this_branch_not_at_the_ee_top_level & ee_specific_files_in_ce_master_not_at_the_ee_top_level
......@@ -59,15 +27,11 @@ new_ee_specific_files_not_at_the_ee_top_level.each do |file|
end
if status.zero?
puts
puts "🎉 All good, congrats! 🎉"
say "🎉 All good, congrats! 🎉"
end
run_git_command("remote remove canonical-ee")
run_git_command("remote remove canonical-ce")
remove_remotes
puts
puts "ℹ️ For more information on the why and how of this job, see https://docs.gitlab.com/ee/development/ee_features.html#detection-of-ee-only-files"
puts
say "ℹ️ For more information on the why and how of this job, see https://docs.gitlab.com/ee/development/ee_features.html#detection-of-ee-only-files"
exit(status)
#!/usr/bin/env ruby
require_relative 'ee_specific_check/ee_specific_check'
include EESpecificCheck # rubocop:disable Style/MixinUsage
base = find_compare_base
current_numstat = updated_diff_numstat(base.ce_merge_base, base.ee_merge_base)
updated_numstat = updated_diff_numstat(base.ce_updated_base, 'HEAD')
offenses = updated_numstat.select do |file, updated_delta|
current_delta = current_numstat[file]
more_lines =
current_delta &&
updated_delta > current_delta &&
WHITELIST.all? { |pattern| !Dir.glob(pattern).include?(file) }
# Don't complain if we're just adding one more line, which could be
# `prepend EE::Module` that we want.
more_lines && !(current_delta == 0 && updated_delta == 1)
end
if offenses.empty?
say "🎉 All good, congrats! 🎉"
else
puts
offenses.each do |(file, delta)|
puts "* 💥 #{file} has #{delta - current_numstat[file]} more different lines than before! 💥"
end
say <<~MESSAGE
ℹ️ Consider using an EE module to add the features you want.
ℹ️ See this for detail: https://docs.gitlab.com/ee/development/ee_features.html#ee-features-based-on-ce-features
MESSAGE
end
remove_remotes
say "ℹ️ For more information on why, see https://gitlab.com/gitlab-org/gitlab-ee/issues/2952"
exit(offenses.size)
module EESpecificCheck
WHITELIST = [
'CHANGELOG-EE.md',
'config/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4946
'doc/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4948#note_59945483
'qa/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/4997#note_59764702
'scripts/**/*',
'spec/javascripts/**/*', # https://gitlab.com/gitlab-org/gitlab-ee/issues/3871
'vendor/assets/javascripts/jasmine-jquery.js',
'.gitlab-ci.yml'
].freeze
CompareBase = Struct.new(:ce_merge_base, :ee_merge_base, :ce_updated_base)
module_function
def say(message)
puts "\n#{message}", "\n" # puts would eat trailing newline
end
def find_compare_base
setup_canonical_remotes
fetch_head_ce = fetch_remote_ce_branch
ce_merge_base = run_git_command("merge-base #{fetch_head_ce} HEAD").strip
ee_merge_base = run_git_command("merge-base canonical-ee/master HEAD").strip
ce_updated_base =
if fetch_head_ce.start_with?('canonical-ce')
ce_merge_base # Compare with merge-base if no specific CE branch
else
fetch_head_ce # Compare with the new HEAD if we do have one
end
CompareBase.new(ce_merge_base, ee_merge_base, ce_updated_base)
end
def setup_canonical_remotes
run_git_command(
"remote add canonical-ee https://gitlab.com/gitlab-org/gitlab-ee.git",
"remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-ce.git")
run_git_command("fetch canonical-ee master --quiet")
end
def fetch_remote_ce_branch
remote_to_fetch, branch_to_fetch = find_remote_ce_branch
run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet")
"#{remote_to_fetch}/#{branch_to_fetch}"
end
def remove_remotes
run_git_command(
"remote remove canonical-ee",
"remote remove canonical-ce",
"remote remove target-ce")
end
def updated_diff_numstat(from, to)
scan_diff_numstat(
run_git_command("diff #{from}..#{to} --numstat -- . '!ee'"))
end
def find_remote_ce_branch
ls_remote_output = run_git_command("ls-remote #{ce_repo_url} \"*#{minimal_ce_branch_name}*\"")
if ls_remote_output.include?(minimal_ce_branch_name)
branch_to_fetch = ls_remote_output.scan(%r{(?<=refs/heads/).+}).sort_by(&:size).first
say "💪 We found the branch '#{branch_to_fetch}' in the #{ce_repo_url} repository. We will fetch it."
run_git_command("remote add target-ce #{ce_repo_url}")
['target-ce', branch_to_fetch]
else
say <<~MESSAGE
⚠️ We did not find a branch that would match the current '#{current_branch}' branch in the #{ce_repo_url} repository. We will fetch 'master' instead.
ℹ️ If you have a CE branch for the current branch, make sure that its name includes '#{minimal_ce_branch_name}'.
MESSAGE
%w[canonical-ce master]
end
end
def ce_repo_url
@ce_repo_url ||= ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-ce.git').sub('gitlab-ee', 'gitlab-ce')
end
def current_branch
@current_branch ||= ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip
end
def minimal_ce_branch_name
@minimal_ce_branch_name ||= current_branch.sub(/(\Aee\-|\-ee\z)/, '')
end
def scan_diff_numstat(numstat)
numstat.scan(/(\d+)\s+(\d+)\s+(.+)/)
.each_with_object({}) do |(added, deleted, file), result|
result[file] = added.to_i + deleted.to_i
end
end
def run_git_command(*commands)
cmds = commands.map { |cmd| "git #{cmd}" }
output = run_command(*cmds)
if commands.size == 1
output.first
else
output
end
end
def run_command(*commands)
commands.map do |cmd|
puts "=> Running `#{cmd}`"
`#{cmd}`
end
end
end
if $0 == __FILE__
require 'rspec/autorun'
RSpec.describe EESpecificCheck do
include EESpecificCheck
before do
allow(self).to receive(:puts)
end
describe '.run_git_command' do
it 'returns the single output when there is a single command' do
output = run_git_command('status')
expect(output).to be_kind_of(String)
expect(self).to have_received(:puts).with(/git status/)
end
it 'returns an array of output for more commands' do
output = run_git_command('status', 'help')
expect(output).to all(be_a(String))
expect(self).to have_received(:puts).with(/git status/)
expect(self).to have_received(:puts).with(/git help/)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment