Save Knapsack reports in S3 instead of in shared cache
Signed-off-by: Rémy Coutable <remy@rymai.me>
This commit is contained in:
parent
ec6f091e27
commit
3504ee1c25
|
@ -14,13 +14,15 @@ variables:
|
|||
GIT_DEPTH: "20"
|
||||
PHANTOMJS_VERSION: "2.1.1"
|
||||
GET_SOURCES_ATTEMPTS: "3"
|
||||
KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-${CI_COMMIT_REF_SLUG}.json
|
||||
KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-${CI_COMMIT_REF_SLUG}.json
|
||||
|
||||
before_script:
|
||||
- source ./scripts/prepare_build.sh
|
||||
- cp config/gitlab.yml.example config/gitlab.yml
|
||||
- bundle --version
|
||||
- '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
|
||||
- retry gem install knapsack
|
||||
- retry gem install knapsack fog-aws mime-types
|
||||
- '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
|
||||
|
||||
stages:
|
||||
|
@ -39,14 +41,15 @@ stages:
|
|||
variables:
|
||||
SETUP_DB: "false"
|
||||
USE_BUNDLE_INSTALL: "false"
|
||||
KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
|
||||
cache:
|
||||
key: "knapsack"
|
||||
paths:
|
||||
- knapsack/
|
||||
- knapsack/
|
||||
artifacts:
|
||||
expire_in: 31d
|
||||
paths:
|
||||
- knapsack/
|
||||
- knapsack/
|
||||
|
||||
.use-db: &use-db
|
||||
services:
|
||||
|
@ -61,17 +64,17 @@ stages:
|
|||
- JOB_NAME=( $CI_JOB_NAME )
|
||||
- export CI_NODE_INDEX=${JOB_NAME[1]}
|
||||
- export CI_NODE_TOTAL=${JOB_NAME[2]}
|
||||
- export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
|
||||
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
|
||||
- export KNAPSACK_GENERATE_REPORT=true
|
||||
- cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
|
||||
- cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
|
||||
- knapsack rspec "--color --format documentation"
|
||||
artifacts:
|
||||
expire_in: 31d
|
||||
when: always
|
||||
paths:
|
||||
- coverage/
|
||||
- knapsack/
|
||||
- tmp/capybara/
|
||||
- coverage/
|
||||
- knapsack/
|
||||
- tmp/capybara/
|
||||
|
||||
.spinach-knapsack: &spinach-knapsack
|
||||
stage: test
|
||||
|
@ -81,28 +84,44 @@ stages:
|
|||
- JOB_NAME=( $CI_JOB_NAME )
|
||||
- export CI_NODE_INDEX=${JOB_NAME[1]}
|
||||
- export CI_NODE_TOTAL=${JOB_NAME[2]}
|
||||
- export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
|
||||
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
|
||||
- export KNAPSACK_GENERATE_REPORT=true
|
||||
- cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
|
||||
- cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
|
||||
- knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
|
||||
artifacts:
|
||||
expire_in: 31d
|
||||
when: always
|
||||
paths:
|
||||
- coverage/
|
||||
- knapsack/
|
||||
- tmp/capybara/
|
||||
- coverage/
|
||||
- knapsack/
|
||||
- tmp/capybara/
|
||||
|
||||
# Prepare and merge knapsack tests
|
||||
|
||||
knapsack:
|
||||
<<: *knapsack-state
|
||||
<<: *dedicated-runner
|
||||
stage: prepare
|
||||
script:
|
||||
- mkdir -p knapsack/
|
||||
- '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
|
||||
- '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
|
||||
- mkdir -p knapsack/${CI_PROJECT_NAME}/
|
||||
- wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || true
|
||||
- wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || true
|
||||
- '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
|
||||
- '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
|
||||
|
||||
update-knapsack:
|
||||
<<: *knapsack-state
|
||||
<<: *dedicated-runner
|
||||
stage: post-test
|
||||
script:
|
||||
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
|
||||
- scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
|
||||
- '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
|
||||
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
|
||||
only:
|
||||
- master@gitlab-org/gitlab-ce
|
||||
- master@gitlab-org/gitlab-ee
|
||||
- master@gitlab/gitlabhq
|
||||
- master@gitlab/gitlab-ee
|
||||
|
||||
setup-test-env:
|
||||
<<: *use-db
|
||||
|
@ -122,20 +141,6 @@ setup-test-env:
|
|||
- public/assets
|
||||
- tmp/tests
|
||||
|
||||
update-knapsack:
|
||||
<<: *knapsack-state
|
||||
<<: *dedicated-runner
|
||||
stage: post-test
|
||||
script:
|
||||
- scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
|
||||
- scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
|
||||
- rm -f knapsack/*_node_*.json
|
||||
only:
|
||||
- master@gitlab-org/gitlab-ce
|
||||
- master@gitlab-org/gitlab-ee
|
||||
- master@gitlab/gitlabhq
|
||||
- master@gitlab/gitlab-ee
|
||||
|
||||
rspec 0 20: *rspec-knapsack
|
||||
rspec 1 20: *rspec-knapsack
|
||||
rspec 2 20: *rspec-knapsack
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
require 'json'
|
||||
require 'yaml'
|
||||
|
||||
main_report_file = ARGV.shift
|
||||
unless main_report_file
|
||||
|
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
require 'rubygems'
|
||||
require 'fog/aws'
|
||||
|
||||
class SyncReports
|
||||
ACTIONS = %w[get put].freeze
|
||||
|
||||
attr_reader :options
|
||||
|
||||
def initialize(options)
|
||||
@options = options
|
||||
|
||||
perform_sync!
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def perform_sync!
|
||||
case options[:action]
|
||||
when 'get'
|
||||
get_reports!
|
||||
when 'put'
|
||||
put_reports!
|
||||
end
|
||||
end
|
||||
|
||||
def get_reports!
|
||||
options[:report_paths].each { |report_path| get_report!(report_path) }
|
||||
end
|
||||
|
||||
def put_reports!
|
||||
options[:report_paths].each { |report_path| put_report!(report_path) }
|
||||
end
|
||||
|
||||
def get_report!(report_path)
|
||||
file = bucket.files.get(report_path)
|
||||
|
||||
if file.respond_to?(:body)
|
||||
File.write(report_path, file.body)
|
||||
puts "#{report_path} was retrieved from S3."
|
||||
else
|
||||
puts "#{report_path} does not seem to exist on S3."
|
||||
end
|
||||
end
|
||||
|
||||
def put_report!(report_path)
|
||||
bucket.files.create(
|
||||
key: report_path,
|
||||
body: File.open(report_path),
|
||||
public: true
|
||||
)
|
||||
puts "#{report_path} was uploaded to S3."
|
||||
end
|
||||
|
||||
def bucket
|
||||
@bucket ||= storage.directories.get(options[:bucket])
|
||||
end
|
||||
|
||||
def storage
|
||||
@storage ||=
|
||||
Fog::Storage.new(
|
||||
provider: 'AWS',
|
||||
aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY']
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def usage!(error: 'action')
|
||||
print "\n[ERROR]: "
|
||||
case error
|
||||
when 'action'
|
||||
puts "Please specify an action as first argument: #{SyncReports::ACTIONS.join(', ')}\n\n"
|
||||
when 'bucket'
|
||||
puts "Please specify a bucket as second argument!\n\n"
|
||||
when 'files'
|
||||
puts "Please specify one or more file paths as third argument!\n\n"
|
||||
end
|
||||
puts "Usage: #{__FILE__} [get|put] bucket report_path ...\n\n"
|
||||
puts "Note: the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment "\
|
||||
"variables need to be set\n\n"
|
||||
exit 1
|
||||
end
|
||||
|
||||
if $0 == __FILE__
|
||||
action = ARGV.shift
|
||||
usage!(error: 'action') unless SyncReports::ACTIONS.include?(action)
|
||||
|
||||
bucket = ARGV.shift
|
||||
usage!(error: 'bucket') unless bucket
|
||||
usage!(error: 'files') unless ARGV.any?
|
||||
|
||||
SyncReports.new(action: action, bucket: bucket, report_paths: ARGV)
|
||||
end
|
Loading…
Reference in New Issue