Commit 3504ee1c authored by Rémy Coutable's avatar Rémy Coutable

Save Knapsack reports in S3 instead of in shared cache

Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent ec6f091e
...@@ -14,13 +14,15 @@ variables: ...@@ -14,13 +14,15 @@ variables:
GIT_DEPTH: "20" GIT_DEPTH: "20"
PHANTOMJS_VERSION: "2.1.1" PHANTOMJS_VERSION: "2.1.1"
GET_SOURCES_ATTEMPTS: "3" GET_SOURCES_ATTEMPTS: "3"
KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json
KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json
before_script: before_script:
- source ./scripts/prepare_build.sh - source ./scripts/prepare_build.sh
- cp config/gitlab.yml.example config/gitlab.yml - cp config/gitlab.yml.example config/gitlab.yml
- bundle --version - bundle --version
- '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS' - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
- retry gem install knapsack - retry gem install knapsack fog-aws mime-types
- '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql' - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
stages: stages:
...@@ -39,14 +41,15 @@ stages: ...@@ -39,14 +41,15 @@ stages:
variables: variables:
SETUP_DB: "false" SETUP_DB: "false"
USE_BUNDLE_INSTALL: "false" USE_BUNDLE_INSTALL: "false"
KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
cache: cache:
key: "knapsack" key: "knapsack"
paths: paths:
- knapsack/ - knapsack/
artifacts: artifacts:
expire_in: 31d expire_in: 31d
paths: paths:
- knapsack/ - knapsack/
.use-db: &use-db .use-db: &use-db
services: services:
...@@ -61,17 +64,17 @@ stages: ...@@ -61,17 +64,17 @@ stages:
- JOB_NAME=( $CI_JOB_NAME ) - JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_INDEX=${JOB_NAME[1]}
- export CI_NODE_TOTAL=${JOB_NAME[2]} - export CI_NODE_TOTAL=${JOB_NAME[2]}
- export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true - export KNAPSACK_GENERATE_REPORT=true
- cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH} - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
- knapsack rspec "--color --format documentation" - knapsack rspec "--color --format documentation"
artifacts: artifacts:
expire_in: 31d expire_in: 31d
when: always when: always
paths: paths:
- coverage/ - coverage/
- knapsack/ - knapsack/
- tmp/capybara/ - tmp/capybara/
.spinach-knapsack: &spinach-knapsack .spinach-knapsack: &spinach-knapsack
stage: test stage: test
...@@ -81,28 +84,44 @@ stages: ...@@ -81,28 +84,44 @@ stages:
- JOB_NAME=( $CI_JOB_NAME ) - JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_INDEX=${JOB_NAME[1]}
- export CI_NODE_TOTAL=${JOB_NAME[2]} - export CI_NODE_TOTAL=${JOB_NAME[2]}
- export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true - export KNAPSACK_GENERATE_REPORT=true
- cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH} - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
- knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)' - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
artifacts: artifacts:
expire_in: 31d expire_in: 31d
when: always when: always
paths: paths:
- coverage/ - coverage/
- knapsack/ - knapsack/
- tmp/capybara/ - tmp/capybara/
# Prepare and merge knapsack tests # Prepare and merge knapsack tests
knapsack: knapsack:
<<: *knapsack-state <<: *knapsack-state
<<: *dedicated-runner <<: *dedicated-runner
stage: prepare stage: prepare
script: script:
- mkdir -p knapsack/ - mkdir -p knapsack/${CI_PROJECT_NAME}/
- '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json' - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true
- '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json' - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true
- '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
- '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
update-knapsack:
<<: *knapsack-state
<<: *dedicated-runner
stage: post-test
script:
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
- scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
- '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- master@gitlab/gitlabhq
- master@gitlab/gitlab-ee
setup-test-env: setup-test-env:
<<: *use-db <<: *use-db
...@@ -122,20 +141,6 @@ setup-test-env: ...@@ -122,20 +141,6 @@ setup-test-env:
- public/assets - public/assets
- tmp/tests - tmp/tests
update-knapsack:
<<: *knapsack-state
<<: *dedicated-runner
stage: post-test
script:
- scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
- scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
- rm -f knapsack/*_node_*.json
only:
- master@gitlab-org/gitlab-ce
- master@gitlab-org/gitlab-ee
- master@gitlab/gitlabhq
- master@gitlab/gitlab-ee
rspec 0 20: *rspec-knapsack rspec 0 20: *rspec-knapsack
rspec 1 20: *rspec-knapsack rspec 1 20: *rspec-knapsack
rspec 2 20: *rspec-knapsack rspec 2 20: *rspec-knapsack
......
#!/usr/bin/env ruby #!/usr/bin/env ruby
require 'json' require 'json'
require 'yaml'
main_report_file = ARGV.shift main_report_file = ARGV.shift
unless main_report_file unless main_report_file
......
#!/usr/bin/env ruby
require 'rubygems'
require 'fog/aws'
class SyncReports
ACTIONS = %w[get put].freeze
attr_reader :options
def initialize(options)
@options = options
perform_sync!
end
private
def perform_sync!
case options[:action]
when 'get'
get_reports!
when 'put'
put_reports!
end
end
def get_reports!
options[:report_paths].each { |report_path| get_report!(report_path) }
end
def put_reports!
options[:report_paths].each { |report_path| put_report!(report_path) }
end
def get_report!(report_path)
file = bucket.files.get(report_path)
if file.respond_to?(:body)
File.write(report_path, file.body)
puts "#{report_path} was retrieved from S3."
else
puts "#{report_path} does not seem to exist on S3."
end
end
def put_report!(report_path)
bucket.files.create(
key: report_path,
body: File.open(report_path),
public: true
)
puts "#{report_path} was uploaded to S3."
end
def bucket
@bucket ||= storage.directories.get(options[:bucket])
end
def storage
@storage ||=
Fog::Storage.new(
provider: 'AWS',
aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
)
end
end
def usage!(error: 'action')
print "\n[ERROR]: "
case error
when 'action'
puts "Please specify an action as first argument: #{SyncReports::ACTIONS.join(', ')}\n\n"
when 'bucket'
puts "Please specify a bucket as second argument!\n\n"
when 'files'
puts "Please specify one or more file paths as third argument!\n\n"
end
puts "Usage: #{__FILE__} [get|put] bucket report_path ...\n\n"
puts "Note: the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment "\
"variables need to be set\n\n"
exit 1
end
if $0 == __FILE__
action = ARGV.shift
usage!(error: 'action') unless SyncReports::ACTIONS.include?(action)
bucket = ARGV.shift
usage!(error: 'bucket') unless bucket
usage!(error: 'files') unless ARGV.any?
SyncReports.new(action: action, bucket: bucket, report_paths: ARGV)
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment