Commit 5956ddd8 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'retry-tests' into 'master'

Retry failed tests

This is to make the CI builds more reliable due to large amount of timeouts and deadlocks recently.


See merge request !1793
parents 500337c4 b651cd22
...@@ -12,6 +12,8 @@ cache: ...@@ -12,6 +12,8 @@ cache:
variables: variables:
MYSQL_ALLOW_EMPTY_PASSWORD: "1" MYSQL_ALLOW_EMPTY_PASSWORD: "1"
# retry tests only in CI environment
RSPEC_RETRY_RETRY_COUNT: "3"
before_script: before_script:
- source ./scripts/prepare_build.sh - source ./scripts/prepare_build.sh
...@@ -21,7 +23,7 @@ before_script: ...@@ -21,7 +23,7 @@ before_script:
- cp config/gitlab.yml.example config/gitlab.yml - cp config/gitlab.yml.example config/gitlab.yml
- touch log/application.log - touch log/application.log
- touch log/test.log - touch log/test.log
- bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}" - bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"
- RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate - RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate
stages: stages:
......
...@@ -263,7 +263,9 @@ group :development, :test do ...@@ -263,7 +263,9 @@ group :development, :test do
gem 'database_cleaner', '~> 1.4.0' gem 'database_cleaner', '~> 1.4.0'
gem 'factory_girl_rails', '~> 4.6.0' gem 'factory_girl_rails', '~> 4.6.0'
gem 'rspec-rails', '~> 3.3.0' gem 'rspec-rails', '~> 3.3.0'
gem 'rspec-retry'
gem 'spinach-rails', '~> 0.2.1' gem 'spinach-rails', '~> 0.2.1'
gem 'spinach-rerun-reporter', '~> 0.0.2'
# Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826) # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
gem 'minitest', '~> 5.7.0' gem 'minitest', '~> 5.7.0'
......
...@@ -679,6 +679,8 @@ GEM ...@@ -679,6 +679,8 @@ GEM
rspec-expectations (~> 3.3.0) rspec-expectations (~> 3.3.0)
rspec-mocks (~> 3.3.0) rspec-mocks (~> 3.3.0)
rspec-support (~> 3.3.0) rspec-support (~> 3.3.0)
rspec-retry (0.4.5)
rspec-core
rspec-support (3.3.0) rspec-support (3.3.0)
rubocop (0.35.1) rubocop (0.35.1)
astrolabe (~> 1.3) astrolabe (~> 1.3)
...@@ -764,6 +766,8 @@ GEM ...@@ -764,6 +766,8 @@ GEM
capybara (>= 2.0.0) capybara (>= 2.0.0)
railties (>= 3) railties (>= 3)
spinach (>= 0.4) spinach (>= 0.4)
spinach-rerun-reporter (0.0.2)
spinach (~> 0.8)
spring (1.6.4) spring (1.6.4)
spring-commands-rspec (1.0.4) spring-commands-rspec (1.0.4)
spring (>= 0.9.1) spring (>= 0.9.1)
...@@ -999,6 +1003,7 @@ DEPENDENCIES ...@@ -999,6 +1003,7 @@ DEPENDENCIES
rouge (~> 1.10.1) rouge (~> 1.10.1)
rqrcode-rails3 (~> 0.1.7) rqrcode-rails3 (~> 0.1.7)
rspec-rails (~> 3.3.0) rspec-rails (~> 3.3.0)
rspec-retry
rubocop (~> 0.35.0) rubocop (~> 0.35.0)
ruby-fogbugz (~> 0.2.1) ruby-fogbugz (~> 0.2.1)
sanitize (~> 2.0) sanitize (~> 2.0)
...@@ -1017,6 +1022,7 @@ DEPENDENCIES ...@@ -1017,6 +1022,7 @@ DEPENDENCIES
six (~> 0.2.0) six (~> 0.2.0)
slack-notifier (~> 1.2.0) slack-notifier (~> 1.2.0)
spinach-rails (~> 0.2.1) spinach-rails (~> 0.2.1)
spinach-rerun-reporter (~> 0.0.2)
spring (~> 1.6.4) spring (~> 1.6.4)
spring-commands-rspec (~> 1.0.4) spring-commands-rspec (~> 1.0.4)
spring-commands-spinach (~> 1.0.0) spring-commands-spinach (~> 1.0.0)
......
...@@ -14,6 +14,7 @@ require 'sidekiq/testing/inline' ...@@ -14,6 +14,7 @@ require 'sidekiq/testing/inline'
require_relative 'capybara' require_relative 'capybara'
require_relative 'db_cleaner' require_relative 'db_cleaner'
require_relative 'rerun'
%w(select2_helper test_env repo_helpers).each do |f| %w(select2_helper test_env repo_helpers).each do |f|
require Rails.root.join('spec', 'support', f) require Rails.root.join('spec', 'support', f)
......
# The spinach-rerun-reporter doesn't define the on_undefined_step
# See it here: https://github.com/javierav/spinach-rerun-reporter/blob/master/lib/spinach/reporter/rerun.rb
module Spinach
class Reporter
class Rerun
def on_undefined_step(step_data, failure, step_definitions = nil)
super step_data, failure, step_definitions
# save feature file and scenario line
@rerun << "#{current_feature.filename}:#{current_scenario.line}"
end
end
end
end
...@@ -4,53 +4,59 @@ namespace :spinach do ...@@ -4,53 +4,59 @@ namespace :spinach do
namespace :project do namespace :project do
desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
task :half do task :half do
cmds = [ run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
%W(rake gitlab:setup),
%W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
]
run_commands(cmds)
end end
desc "GitLab | Spinach | Run remaining project spinach features" desc "GitLab | Spinach | Run remaining project spinach features"
task :rest do task :rest do
cmds = [ run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
%W(rake gitlab:setup),
%W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
]
run_commands(cmds)
end end
end end
desc "GitLab | Spinach | Run project spinach features" desc "GitLab | Spinach | Run project spinach features"
task :project do task :project do
cmds = [ run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
%W(rake gitlab:setup),
%W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
]
run_commands(cmds)
end end
desc "GitLab | Spinach | Run other spinach features" desc "GitLab | Spinach | Run other spinach features"
task :other do task :other do
cmds = [ run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
%W(rake gitlab:setup), end
%W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
] desc "GitLab | Spinach | Run other spinach features"
run_commands(cmds) task :builds do
run_spinach_tests('@builds')
end end
end end
desc "GitLab | Run spinach" desc "GitLab | Run spinach"
task :spinach do task :spinach do
cmds = [ run_spinach_tests(nil)
%W(rake gitlab:setup), end
%W(spinach),
] def run_command(cmd)
run_commands(cmds) system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
end end
def run_commands(cmds) def run_spinach_command(args)
cmds.each do |cmd| run_command(%w(spinach -r rerun) + args)
system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end
def run_spinach_tests(tags)
#run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
success = run_spinach_command(%W(--tags #{tags}))
3.times do |_|
break if success
break unless File.exists?('tmp/spinach-rerun.txt')
tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
puts ''
puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red
puts ''
sleep(3)
success = run_spinach_command(tests)
end end
raise("spinach tests for #{tags} failed!") unless success
end end
...@@ -2,15 +2,27 @@ ...@@ -2,15 +2,27 @@
if [ -f /.dockerinit ]; then if [ -f /.dockerinit ]; then
mkdir -p vendor mkdir -p vendor
if [ ! -e vendor/phantomjs_1.9.8-0jessie_amd64.deb ]; then
# Install phantomjs package
pushd vendor
if [ ! -e phantomjs_1.9.8-0jessie_amd64.deb ]; then
wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb
mv phantomjs_1.9.8-0jessie_amd64.deb vendor/
fi fi
dpkg -i vendor/phantomjs_1.9.8-0jessie_amd64.deb dpkg -i phantomjs_1.9.8-0jessie_amd64.deb
popd
# Try to install packages
for i in $(seq 1 3); do
apt-get update -yqqq || true
if apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \
libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip; then
break
fi
apt-get update -qq sleep 3s
apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \ echo "Retrying package installation..."
libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip done
cp config/database.yml.mysql config/database.yml cp config/database.yml.mysql config/database.yml
sed -i 's/username:.*/username: root/g' config/database.yml sed -i 's/username:.*/username: root/g' config/database.yml
...@@ -20,7 +32,7 @@ if [ -f /.dockerinit ]; then ...@@ -20,7 +32,7 @@ if [ -f /.dockerinit ]; then
cp config/resque.yml.example config/resque.yml cp config/resque.yml.example config/resque.yml
sed -i 's/localhost/redis/g' config/resque.yml sed -i 's/localhost/redis/g' config/resque.yml
export FLAGS=(--path vendor) export FLAGS=(--path vendor --retry 3)
else else
export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin
cp config/database.yml.mysql config/database.yml cp config/database.yml.mysql config/database.yml
......
...@@ -15,6 +15,7 @@ require 'rspec/rails' ...@@ -15,6 +15,7 @@ require 'rspec/rails'
require 'shoulda/matchers' require 'shoulda/matchers'
require 'sidekiq/testing/inline' require 'sidekiq/testing/inline'
require 'benchmark/ips' require 'benchmark/ips'
require 'rspec/retry'
# Requires supporting ruby files with custom matchers and macros, etc, # Requires supporting ruby files with custom matchers and macros, etc,
# in spec/support/ and its subdirectories. # in spec/support/ and its subdirectories.
...@@ -25,6 +26,9 @@ RSpec.configure do |config| ...@@ -25,6 +26,9 @@ RSpec.configure do |config|
config.use_instantiated_fixtures = false config.use_instantiated_fixtures = false
config.mock_with :rspec config.mock_with :rspec
config.verbose_retry = true
config.display_try_failure_messages = true
config.include Devise::TestHelpers, type: :controller config.include Devise::TestHelpers, type: :controller
config.include LoginHelpers, type: :feature config.include LoginHelpers, type: :feature
config.include LoginHelpers, type: :request config.include LoginHelpers, type: :request
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment