2020-05-18 12:08:08 +00:00
#!/usr/bin/env bash
2019-09-27 15:06:16 +00:00
function retrieve_tests_metadata( ) {
2021-08-30 12:09:48 +00:00
mkdir -p $( dirname " $KNAPSACK_RSPEC_SUITE_REPORT_PATH " ) $( dirname " $FLAKY_RSPEC_SUITE_REPORT_PATH " ) rspec_profiling/
2020-11-25 15:09:13 +00:00
2021-08-30 12:09:48 +00:00
if [ [ -n " ${ RETRIEVE_TESTS_METADATA_FROM_PAGES } " ] ] ; then
if [ [ ! -f " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " ] ] ; then
curl --location -o " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " " https://gitlab-org.gitlab.io/gitlab/ ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " || echo "{}" > " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } "
fi
2020-11-25 15:09:13 +00:00
2021-08-30 12:09:48 +00:00
if [ [ ! -f " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " ] ] ; then
curl --location -o " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " " https://gitlab-org.gitlab.io/gitlab/ ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " || echo "{}" > " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } "
fi
else
# ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
# always target the canonical project here, so the branch must be hardcoded
local project_path = "gitlab-org/gitlab"
local artifact_branch = "master"
2021-11-25 00:10:49 +00:00
local username = "gitlab-bot"
local job_name = "update-tests-metadata"
2021-08-30 12:09:48 +00:00
local test_metadata_job_id
# Ruby
2021-11-25 00:10:49 +00:00
test_metadata_job_id = $( scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " -q "status=success" -q " ref= ${ artifact_branch } " -q " username= ${ username } " -Q "scope=success" --job-name " ${ job_name } " )
if [ [ -n " ${ test_metadata_job_id } " ] ] ; then
echo " test_metadata_job_id: ${ test_metadata_job_id } "
if [ [ ! -f " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " ] ] ; then
scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " --job-id " ${ test_metadata_job_id } " --artifact-path " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " || echo "{}" > " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } "
fi
if [ [ ! -f " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " ] ] ; then
scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " --job-id " ${ test_metadata_job_id } " --artifact-path " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " || echo "{}" > " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } "
fi
else
echo "test_metadata_job_id couldn't be found!"
echo "{}" > " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } "
echo "{}" > " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } "
2021-08-30 12:09:48 +00:00
fi
2019-09-27 15:06:16 +00:00
fi
}
function update_tests_metadata( ) {
echo "{}" > " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } "
2020-04-21 21:09:38 +00:00
scripts/merge-reports " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " knapsack/rspec*.json
2019-10-01 12:05:59 +00:00
rm -f knapsack/rspec*.json
2019-09-27 15:06:16 +00:00
2020-11-24 09:09:32 +00:00
export FLAKY_RSPEC_GENERATE_REPORT = "true"
2020-11-25 15:09:13 +00:00
scripts/merge-reports " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } " rspec_flaky/all_*.json
2020-01-21 21:08:54 +00:00
scripts/flaky_examples/prune-old-flaky-examples " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } "
2019-10-01 12:05:59 +00:00
rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
2019-09-27 15:06:16 +00:00
2020-04-21 21:09:38 +00:00
if [ [ " $CI_PIPELINE_SOURCE " = = "schedule" ] ] ; then
scripts/insert-rspec-profiling-data
else
echo "Not inserting profiling data as the pipeline is not a scheduled one."
fi
2019-09-27 15:06:16 +00:00
}
2020-12-09 15:10:12 +00:00
function retrieve_tests_mapping( ) {
2021-08-30 12:09:48 +00:00
mkdir -p $( dirname " $RSPEC_PACKED_TESTS_MAPPING_PATH " )
2020-12-09 15:10:12 +00:00
2021-08-30 12:09:48 +00:00
if [ [ -n " ${ RETRIEVE_TESTS_METADATA_FROM_PAGES } " ] ] ; then
if [ [ ! -f " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } " ] ] ; then
( curl --location -o " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } .gz " " https://gitlab-org.gitlab.io/gitlab/ ${ RSPEC_PACKED_TESTS_MAPPING_PATH } .gz " && gzip -d " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } .gz " ) || echo "{}" > " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
fi
else
# ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
# always target the canonical project here, so the branch must be hardcoded
local project_path = "gitlab-org/gitlab"
local artifact_branch = "master"
2021-11-25 00:10:49 +00:00
local username = "gitlab-bot"
local job_name = "update-tests-metadata"
2021-08-30 12:09:48 +00:00
local test_metadata_with_mapping_job_id
2020-12-09 15:10:12 +00:00
2021-11-25 00:10:49 +00:00
test_metadata_with_mapping_job_id = $( scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " -q "status=success" -q " ref= ${ artifact_branch } " -q " username= ${ username } " -Q "scope=success" --job-name " ${ job_name } " )
2020-12-09 15:10:12 +00:00
2021-11-25 00:10:49 +00:00
if [ [ -n " ${ test_metadata_with_mapping_job_id } " ] ] ; then
echo " test_metadata_with_mapping_job_id: ${ test_metadata_with_mapping_job_id } "
if [ [ ! -f " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } " ] ] ; then
( scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " --job-id " ${ test_metadata_with_mapping_job_id } " --artifact-path " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } .gz " && gzip -d " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } .gz " ) || echo "{}" > " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
fi
else
echo "test_metadata_with_mapping_job_id couldn't be found!"
echo "{}" > " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
2021-08-30 12:09:48 +00:00
fi
2020-12-09 15:10:12 +00:00
fi
scripts/unpack-test-mapping " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } " " ${ RSPEC_TESTS_MAPPING_PATH } "
}
2021-11-25 00:10:49 +00:00
function retrieve_frontend_fixtures_mapping( ) {
mkdir -p $( dirname " $FRONTEND_FIXTURES_MAPPING_PATH " )
if [ [ -n " ${ RETRIEVE_TESTS_METADATA_FROM_PAGES } " ] ] ; then
if [ [ ! -f " ${ FRONTEND_FIXTURES_MAPPING_PATH } " ] ] ; then
( curl --location -o " ${ FRONTEND_FIXTURES_MAPPING_PATH } " " https://gitlab-org.gitlab.io/gitlab/ ${ FRONTEND_FIXTURES_MAPPING_PATH } " ) || echo "{}" > " ${ FRONTEND_FIXTURES_MAPPING_PATH } "
fi
else
# ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
# always target the canonical project here, so the branch must be hardcoded
local project_path = "gitlab-org/gitlab"
local artifact_branch = "master"
local username = "gitlab-bot"
local job_name = "generate-frontend-fixtures-mapping"
local test_metadata_with_mapping_job_id
# On the MR that introduces 'generate-frontend-fixtures-mapping', we cannot retrieve the file from a master scheduled pipeline, so we take it from a known MR pipeline
if [ [ " ${ CI_MERGE_REQUEST_SOURCE_BRANCH_NAME } " = = "339343-execute-related-jests-specs-for-mrs-with-backend-changes" ] ] ; then
test_metadata_with_mapping_job_id = $( scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " --pipeline-id "414921396" -Q "scope=success" --job-name " ${ job_name } " )
else
test_metadata_with_mapping_job_id = $( scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " -q " ref= ${ artifact_branch } " -q " username= ${ username } " -Q "scope=success" --job-name " ${ job_name } " )
fi
if [ [ $? -eq 0 ] ] && [ [ -n " ${ test_metadata_with_mapping_job_id } " ] ] ; then
echo " test_metadata_with_mapping_job_id: ${ test_metadata_with_mapping_job_id } "
if [ [ ! -f " ${ FRONTEND_FIXTURES_MAPPING_PATH } " ] ] ; then
( scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project " ${ project_path } " --job-id " ${ test_metadata_with_mapping_job_id } " --artifact-path " ${ FRONTEND_FIXTURES_MAPPING_PATH } " ) || echo "{}" > " ${ FRONTEND_FIXTURES_MAPPING_PATH } "
fi
else
echo "test_metadata_with_mapping_job_id couldn't be found!"
echo "{}" > " ${ FRONTEND_FIXTURES_MAPPING_PATH } "
fi
fi
}
2020-10-28 15:08:49 +00:00
function update_tests_mapping( ) {
if ! crystalball_rspec_data_exists; then
echo "No crystalball rspec data found."
return 0
fi
scripts/generate-test-mapping " ${ RSPEC_TESTS_MAPPING_PATH } " crystalball/rspec*.yml
scripts/pack-test-mapping " ${ RSPEC_TESTS_MAPPING_PATH } " " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
gzip " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
2020-11-25 15:09:13 +00:00
rm -f crystalball/rspec*.yml " ${ RSPEC_PACKED_TESTS_MAPPING_PATH } "
2020-10-28 15:08:49 +00:00
}
function crystalball_rspec_data_exists( ) {
2020-11-26 15:09:30 +00:00
compgen -G "crystalball/rspec*.yml" >/dev/null
2020-10-28 15:08:49 +00:00
}
2021-10-25 15:12:11 +00:00
function retrieve_previous_failed_tests( ) {
local directory_for_output_reports = " ${ 1 } "
local rspec_pg_regex = " ${ 2 } "
local rspec_ee_pg_regex = " ${ 3 } "
local pipeline_report_path = "test_results/previous/test_reports.json"
2021-11-03 21:10:35 +00:00
# Used to query merge requests. This variable reflects where the merge request has been created
local target_project_path = " ${ CI_MERGE_REQUEST_PROJECT_PATH } "
local instance_url = " ${ CI_SERVER_URL } "
2021-10-25 15:12:11 +00:00
echo 'Attempting to build pipeline test report...'
2021-11-03 21:10:35 +00:00
scripts/pipeline_test_report_builder.rb --instance-base-url " ${ instance_url } " --target-project " ${ target_project_path } " --mr-id " ${ CI_MERGE_REQUEST_IID } " --output-file-path " ${ pipeline_report_path } "
2021-10-25 15:12:11 +00:00
echo 'Generating failed tests lists...'
scripts/failed_tests.rb --previous-tests-report-path " ${ pipeline_report_path } " --output-directory " ${ directory_for_output_reports } " --rspec-pg-regex " ${ rspec_pg_regex } " --rspec-ee-pg-regex " ${ rspec_ee_pg_regex } "
}
2019-09-27 15:06:16 +00:00
function rspec_simple_job( ) {
local rspec_opts = " ${ 1 } "
export NO_KNAPSACK = "1"
2021-11-25 00:10:49 +00:00
eval " bin/rspec -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${ rspec_opts } "
2019-09-27 15:06:16 +00:00
}
2021-01-20 09:10:52 +00:00
function rspec_db_library_code( ) {
2021-10-08 06:11:46 +00:00
local db_files = "spec/lib/gitlab/database/"
2021-01-20 09:10:52 +00:00
rspec_simple_job " -- ${ db_files } "
}
2019-09-27 15:06:16 +00:00
function rspec_paralellized_job( ) {
2020-04-21 21:09:38 +00:00
read -ra job_name <<< " ${ CI_JOB_NAME } "
2019-09-27 15:06:16 +00:00
local test_tool = " ${ job_name [0] } "
local test_level = " ${ job_name [1] } "
2021-05-10 12:10:26 +00:00
local report_name = $( echo " ${ CI_JOB_NAME } " | sed -E 's|[/ ]|_|g' ) # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24'
2019-09-27 15:06:16 +00:00
local rspec_opts = " ${ 1 } "
2021-10-04 18:12:46 +00:00
local spec_folder_prefixes = ""
2019-09-27 15:06:16 +00:00
if [ [ " ${ test_tool } " = ~ "-ee" ] ] ; then
2021-10-04 18:12:46 +00:00
spec_folder_prefixes = "'ee/'"
2019-09-27 15:06:16 +00:00
fi
2021-04-13 18:11:28 +00:00
if [ [ " ${ test_tool } " = ~ "-jh" ] ] ; then
2021-10-04 18:12:46 +00:00
spec_folder_prefixes = "'jh/'"
fi
if [ [ " ${ test_tool } " = ~ "-all" ] ] ; then
2021-10-13 21:09:56 +00:00
spec_folder_prefixes = "['', 'ee/', 'jh/']"
2021-04-13 18:11:28 +00:00
fi
2019-09-27 15:06:16 +00:00
export KNAPSACK_LOG_LEVEL = "debug"
2020-04-21 21:09:38 +00:00
export KNAPSACK_REPORT_PATH = " knapsack/ ${ report_name } _report.json "
2019-09-27 15:06:16 +00:00
2020-05-08 09:09:39 +00:00
# There's a bug where artifacts are sometimes not downloaded. Since specs can run without the Knapsack report, we can
# handle the missing artifact gracefully here. See https://gitlab.com/gitlab-org/gitlab/-/issues/212349.
if [ [ ! -f " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " ] ] ; then
echo "{}" > " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } "
fi
2019-09-30 09:06:31 +00:00
cp " ${ KNAPSACK_RSPEC_SUITE_REPORT_PATH } " " ${ KNAPSACK_REPORT_PATH } "
2019-09-27 15:06:16 +00:00
if [ [ -z " ${ KNAPSACK_TEST_FILE_PATTERN } " ] ] ; then
2021-10-04 18:12:46 +00:00
pattern = $( ruby -r./tooling/quality/test_level.rb -e " puts Quality::TestLevel.new( ${ spec_folder_prefixes } ).pattern(: ${ test_level } ) " )
2019-09-27 15:06:16 +00:00
export KNAPSACK_TEST_FILE_PATTERN = " ${ pattern } "
fi
echo " KNAPSACK_TEST_FILE_PATTERN: ${ KNAPSACK_TEST_FILE_PATTERN } "
2021-11-17 12:11:55 +00:00
echo " SKIP_FLAKY_TESTS_AUTOMATICALLY: ${ SKIP_FLAKY_TESTS_AUTOMATICALLY } "
2019-09-27 15:06:16 +00:00
if [ [ -d "ee/" ] ] ; then
export KNAPSACK_GENERATE_REPORT = "true"
export FLAKY_RSPEC_GENERATE_REPORT = "true"
export SUITE_FLAKY_RSPEC_REPORT_PATH = " ${ FLAKY_RSPEC_SUITE_REPORT_PATH } "
2020-04-21 21:09:38 +00:00
export FLAKY_RSPEC_REPORT_PATH = " rspec_flaky/all_ ${ report_name } _report.json "
export NEW_FLAKY_RSPEC_REPORT_PATH = " rspec_flaky/new_ ${ report_name } _report.json "
2021-11-08 15:13:35 +00:00
export SKIPPED_FLAKY_TESTS_REPORT_PATH = " rspec_flaky/skipped_flaky_tests_ ${ report_name } _report.txt "
2019-09-27 15:06:16 +00:00
if [ [ ! -f $FLAKY_RSPEC_REPORT_PATH ] ] ; then
echo "{}" > " ${ FLAKY_RSPEC_REPORT_PATH } "
fi
if [ [ ! -f $NEW_FLAKY_RSPEC_REPORT_PATH ] ] ; then
echo "{}" > " ${ NEW_FLAKY_RSPEC_REPORT_PATH } "
fi
fi
mkdir -p tmp/memory_test
2020-04-21 21:09:38 +00:00
export MEMORY_TEST_PATH = " tmp/memory_test/ ${ report_name } _memory.csv "
2019-09-27 15:06:16 +00:00
2020-11-26 15:09:30 +00:00
local rspec_args = " -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${ rspec_opts } "
2021-09-07 09:11:43 +00:00
if [ [ -n $RSPEC_TESTS_MAPPING_ENABLED ] ] ; then
2020-12-09 15:10:12 +00:00
tooling/bin/parallel_rspec --rspec_args " ${ rspec_args } " --filter "tmp/matching_tests.txt"
2020-11-26 15:09:30 +00:00
else
tooling/bin/parallel_rspec --rspec_args " ${ rspec_args } "
fi
2019-09-27 15:06:16 +00:00
date
}
2020-06-16 12:09:00 +00:00
2021-10-25 15:12:11 +00:00
function rspec_rerun_previous_failed_tests( ) {
local test_file_count_threshold = ${ RSPEC_PREVIOUS_FAILED_TEST_FILE_COUNT_THRESHOLD :- 10 }
local matching_tests_file = ${ 1 }
local rspec_opts = ${ 2 }
local test_files = " $( cat " ${ matching_tests_file } " ) "
local test_file_count = $( wc -w " ${ matching_tests_file } " | awk { 'print $1' } )
if [ [ " ${ test_file_count } " -gt " ${ test_file_count_threshold } " ] ] ; then
2021-10-26 15:09:27 +00:00
echo " This job is intentionally exited because there are more than ${ test_file_count_threshold } test files to rerun. "
exit 0
2021-10-25 15:12:11 +00:00
fi
if [ [ -n $test_files ] ] ; then
rspec_simple_job " ${ test_files } "
else
echo "No failed test files to rerun"
fi
}
2020-09-14 15:09:28 +00:00
function rspec_fail_fast( ) {
local test_file_count_threshold = ${ RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD :- 10 }
local matching_tests_file = ${ 1 }
local rspec_opts = ${ 2 }
local test_files = " $( cat " ${ matching_tests_file } " ) "
local test_file_count = $( wc -w " ${ matching_tests_file } " | awk { 'print $1' } )
if [ [ " ${ test_file_count } " -gt " ${ test_file_count_threshold } " ] ] ; then
echo " This job is intentionally skipped because there are more than ${ test_file_count_threshold } test files matched, "
echo "which would take too long to run in this job."
echo "All the tests would be run in other rspec jobs."
exit 0
fi
if [ [ -n $test_files ] ] ; then
rspec_simple_job " ${ rspec_opts } ${ test_files } "
else
echo "No rspec fail-fast tests to run"
fi
}
2020-08-25 15:10:17 +00:00
function rspec_matched_foss_tests( ) {
2020-06-16 12:09:00 +00:00
local test_file_count_threshold = 20
local matching_tests_file = ${ 1 }
local rspec_opts = ${ 2 }
local test_files = " $( cat " ${ matching_tests_file } " ) "
local test_file_count = $( wc -w " ${ matching_tests_file } " | awk { 'print $1' } )
if [ [ " ${ test_file_count } " -gt " ${ test_file_count_threshold } " ] ] ; then
2020-06-17 09:08:38 +00:00
echo " This job is intentionally failed because there are more than ${ test_file_count_threshold } FOSS test files matched, "
2020-06-16 12:09:00 +00:00
echo "which would take too long to run in this job."
echo "To reduce the likelihood of breaking FOSS pipelines,"
2021-08-30 18:10:36 +00:00
echo "please add ~\"pipeline:run-as-if-foss\" label to the merge request and trigger a new pipeline."
2020-06-16 12:09:00 +00:00
echo "This would run all as-if-foss jobs in this merge request"
2020-06-17 09:08:38 +00:00
echo "and remove this failing job from the pipeline."
2020-06-16 12:09:00 +00:00
exit 1
fi
if [ [ -n $test_files ] ] ; then
rspec_simple_job " ${ rspec_opts } ${ test_files } "
else
2020-09-14 15:09:28 +00:00
echo "No impacted FOSS rspec tests to run"
2020-06-16 12:09:00 +00:00
fi
}
2021-11-25 00:10:49 +00:00
function generate_frontend_fixtures_mapping( ) {
local pattern = ""
if [ [ -d "ee/" ] ] ; then
pattern = ",ee/"
fi
if [ [ -d "jh/" ] ] ; then
pattern = " ${ pattern } ,jh/ "
fi
if [ [ -n " ${ pattern } " ] ] ; then
pattern = " { ${ pattern } } "
fi
pattern = " ${ pattern } spec/frontend/fixtures/**/*.rb "
export GENERATE_FRONTEND_FIXTURES_MAPPING = "true"
mkdir -p $( dirname " $FRONTEND_FIXTURES_MAPPING_PATH " )
rspec_simple_job " --pattern \" ${ pattern } \" "
}