Commit a5647149 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '29772-store-knapsack-reports-in-s3' into 'master'

Save Knapsack reports in S3 instead of in shared cache

Closes #29772

See merge request !10192
parents 93998d6e 3504ee1c
......@@ -14,13 +14,15 @@ variables:
GIT_DEPTH: "20"
PHANTOMJS_VERSION: "2.1.1"
GET_SOURCES_ATTEMPTS: "3"
KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json
KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json
before_script:
- source ./scripts/prepare_build.sh
- cp config/gitlab.yml.example config/gitlab.yml
- bundle --version
- '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
- retry gem install knapsack
- retry gem install knapsack fog-aws mime-types
- '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
stages:
......@@ -39,6 +41,7 @@ stages:
variables:
SETUP_DB: "false"
USE_BUNDLE_INSTALL: "false"
KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
cache:
key: "knapsack"
paths:
......@@ -61,9 +64,9 @@ stages:
- JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]}
- export CI_NODE_TOTAL=${JOB_NAME[2]}
- export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true
- cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
- cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
- knapsack rspec "--color --format documentation"
artifacts:
expire_in: 31d
......@@ -81,9 +84,9 @@ stages:
- JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]}
- export CI_NODE_TOTAL=${JOB_NAME[2]}
- export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true
- cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
- cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
- knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
artifacts:
expire_in: 31d
......@@ -94,15 +97,31 @@ stages:
- tmp/capybara/
# Prepare and merge knapsack tests
knapsack:
<<: *knapsack-state
<<: *dedicated-runner
stage: prepare
script:
- mkdir -p knapsack/
- '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
- '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
- mkdir -p knapsack/${CI_PROJECT_NAME}/
- wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true
- wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true
- '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
- '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
update-knapsack:
<<: *knapsack-state
<<: *dedicated-runner
stage: post-test
script:
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
- scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
- '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- master@gitlab/gitlabhq
- master@gitlab/gitlab-ee
setup-test-env:
<<: *use-db
......@@ -122,20 +141,6 @@ setup-test-env:
- public/assets
- tmp/tests
update-knapsack:
<<: *knapsack-state
<<: *dedicated-runner
stage: post-test
script:
- scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
- scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
- rm -f knapsack/*_node_*.json
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- master@gitlab/gitlabhq
- master@gitlab/gitlab-ee
rspec 0 20: *rspec-knapsack
rspec 1 20: *rspec-knapsack
rspec 2 20: *rspec-knapsack
......
#!/usr/bin/env ruby
require 'json'
require 'yaml'
main_report_file = ARGV.shift
unless main_report_file
......
#!/usr/bin/env ruby
require 'rubygems'
require 'fog/aws'
class SyncReports
ACTIONS = %w[get put].freeze
attr_reader :options
def initialize(options)
@options = options
perform_sync!
end
private
def perform_sync!
case options[:action]
when 'get'
get_reports!
when 'put'
put_reports!
end
end
def get_reports!
options[:report_paths].each { |report_path| get_report!(report_path) }
end
def put_reports!
options[:report_paths].each { |report_path| put_report!(report_path) }
end
def get_report!(report_path)
file = bucket.files.get(report_path)
if file.respond_to?(:body)
File.write(report_path, file.body)
puts "#{report_path} was retrieved from S3."
else
puts "#{report_path} does not seem to exist on S3."
end
end
def put_report!(report_path)
bucket.files.create(
key: report_path,
body: File.open(report_path),
public: true
)
puts "#{report_path} was uploaded to S3."
end
def bucket
@bucket ||= storage.directories.get(options[:bucket])
end
def storage
@storage ||=
Fog::Storage.new(
provider: 'AWS',
aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
)
end
end
def usage!(error: 'action')
print "\n[ERROR]: "
case error
when 'action'
puts "Please specify an action as first argument: #{SyncReports::ACTIONS.join(', ')}\n\n"
when 'bucket'
puts "Please specify a bucket as second argument!\n\n"
when 'files'
puts "Please specify one or more file paths as third argument!\n\n"
end
puts "Usage: #{__FILE__} [get|put] bucket report_path ...\n\n"
puts "Note: the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment "\
"variables need to be set\n\n"
exit 1
end
if $0 == __FILE__
action = ARGV.shift
usage!(error: 'action') unless SyncReports::ACTIONS.include?(action)
bucket = ARGV.shift
usage!(error: 'bucket') unless bucket
usage!(error: 'files') unless ARGV.any?
SyncReports.new(action: action, bucket: bucket, report_paths: ARGV)
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment