Commit 5956ddd8 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'retry-tests' into 'master'

Retry failed tests

This is to make the CI builds more reliable due to large amount of timeouts and deadlocks recently.


See merge request !1793
parents 500337c4 b651cd22
......@@ -12,6 +12,8 @@ cache:
variables:
MYSQL_ALLOW_EMPTY_PASSWORD: "1"
# retry tests only in CI environment
RSPEC_RETRY_RETRY_COUNT: "3"
before_script:
- source ./scripts/prepare_build.sh
......
......@@ -263,7 +263,9 @@ group :development, :test do
gem 'database_cleaner', '~> 1.4.0'
gem 'factory_girl_rails', '~> 4.6.0'
gem 'rspec-rails', '~> 3.3.0'
gem 'rspec-retry'
gem 'spinach-rails', '~> 0.2.1'
gem 'spinach-rerun-reporter', '~> 0.0.2'
# Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
gem 'minitest', '~> 5.7.0'
......
......@@ -679,6 +679,8 @@ GEM
rspec-expectations (~> 3.3.0)
rspec-mocks (~> 3.3.0)
rspec-support (~> 3.3.0)
rspec-retry (0.4.5)
rspec-core
rspec-support (3.3.0)
rubocop (0.35.1)
astrolabe (~> 1.3)
......@@ -764,6 +766,8 @@ GEM
capybara (>= 2.0.0)
railties (>= 3)
spinach (>= 0.4)
spinach-rerun-reporter (0.0.2)
spinach (~> 0.8)
spring (1.6.4)
spring-commands-rspec (1.0.4)
spring (>= 0.9.1)
......@@ -999,6 +1003,7 @@ DEPENDENCIES
rouge (~> 1.10.1)
rqrcode-rails3 (~> 0.1.7)
rspec-rails (~> 3.3.0)
rspec-retry
rubocop (~> 0.35.0)
ruby-fogbugz (~> 0.2.1)
sanitize (~> 2.0)
......@@ -1017,6 +1022,7 @@ DEPENDENCIES
six (~> 0.2.0)
slack-notifier (~> 1.2.0)
spinach-rails (~> 0.2.1)
spinach-rerun-reporter (~> 0.0.2)
spring (~> 1.6.4)
spring-commands-rspec (~> 1.0.4)
spring-commands-spinach (~> 1.0.0)
......
......@@ -14,6 +14,7 @@ require 'sidekiq/testing/inline'
require_relative 'capybara'
require_relative 'db_cleaner'
require_relative 'rerun'
%w(select2_helper test_env repo_helpers).each do |f|
require Rails.root.join('spec', 'support', f)
......
# The spinach-rerun-reporter doesn't define the on_undefined_step
# See it here: https://github.com/javierav/spinach-rerun-reporter/blob/master/lib/spinach/reporter/rerun.rb
module Spinach
class Reporter
class Rerun
def on_undefined_step(step_data, failure, step_definitions = nil)
super step_data, failure, step_definitions
# save feature file and scenario line
@rerun << "#{current_feature.filename}:#{current_scenario.line}"
end
end
end
end
......@@ -4,53 +4,59 @@ namespace :spinach do
namespace :project do
desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
task :half do
cmds = [
%W(rake gitlab:setup),
%W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
]
run_commands(cmds)
run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
end
desc "GitLab | Spinach | Run remaining project spinach features"
task :rest do
cmds = [
%W(rake gitlab:setup),
%W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
]
run_commands(cmds)
run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
end
end
desc "GitLab | Spinach | Run project spinach features"
task :project do
cmds = [
%W(rake gitlab:setup),
%W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
]
run_commands(cmds)
run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
end
desc "GitLab | Spinach | Run other spinach features"
task :other do
cmds = [
%W(rake gitlab:setup),
%W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
]
run_commands(cmds)
run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
end
desc "GitLab | Spinach | Run other spinach features"
task :builds do
run_spinach_tests('@builds')
end
end
desc "GitLab | Run spinach"
task :spinach do
cmds = [
%W(rake gitlab:setup),
%W(spinach),
]
run_commands(cmds)
run_spinach_tests(nil)
end
def run_command(cmd)
system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
end
def run_commands(cmds)
cmds.each do |cmd|
system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
def run_spinach_command(args)
run_command(%w(spinach -r rerun) + args)
end
def run_spinach_tests(tags)
#run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
success = run_spinach_command(%W(--tags #{tags}))
3.times do |_|
break if success
break unless File.exists?('tmp/spinach-rerun.txt')
tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
puts ''
puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red
puts ''
sleep(3)
success = run_spinach_command(tests)
end
raise("spinach tests for #{tags} failed!") unless success
end
......@@ -2,15 +2,27 @@
if [ -f /.dockerinit ]; then
mkdir -p vendor
if [ ! -e vendor/phantomjs_1.9.8-0jessie_amd64.deb ]; then
# Install phantomjs package
pushd vendor
if [ ! -e phantomjs_1.9.8-0jessie_amd64.deb ]; then
wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb
mv phantomjs_1.9.8-0jessie_amd64.deb vendor/
fi
dpkg -i vendor/phantomjs_1.9.8-0jessie_amd64.deb
dpkg -i phantomjs_1.9.8-0jessie_amd64.deb
popd
# Try to install packages
for i in $(seq 1 3); do
apt-get update -yqqq || true
if apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \
libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip; then
break
fi
apt-get update -qq
apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \
libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip
sleep 3s
echo "Retrying package installation..."
done
cp config/database.yml.mysql config/database.yml
sed -i 's/username:.*/username: root/g' config/database.yml
......@@ -20,7 +32,7 @@ if [ -f /.dockerinit ]; then
cp config/resque.yml.example config/resque.yml
sed -i 's/localhost/redis/g' config/resque.yml
export FLAGS=(--path vendor)
export FLAGS=(--path vendor --retry 3)
else
export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin
cp config/database.yml.mysql config/database.yml
......
......@@ -15,6 +15,7 @@ require 'rspec/rails'
require 'shoulda/matchers'
require 'sidekiq/testing/inline'
require 'benchmark/ips'
require 'rspec/retry'
# Requires supporting ruby files with custom matchers and macros, etc,
# in spec/support/ and its subdirectories.
......@@ -25,6 +26,9 @@ RSpec.configure do |config|
config.use_instantiated_fixtures = false
config.mock_with :rspec
config.verbose_retry = true
config.display_try_failure_messages = true
config.include Devise::TestHelpers, type: :controller
config.include LoginHelpers, type: :feature
config.include LoginHelpers, type: :request
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment