diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3a97777e99f..71ddebef662 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -364,11 +364,11 @@ update-tests-metadata:
       - rspec_flaky/
     policy: push
   script:
-    - retry gem install fog-aws mime-types
+    - retry gem install fog-aws mime-types activesupport
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
-    - scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
+    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
diff --git a/scripts/prune-old-flaky-specs b/scripts/prune-old-flaky-specs
index ab03a0230cc..f7451fbd428 100755
--- a/scripts/prune-old-flaky-specs
+++ b/scripts/prune-old-flaky-specs
@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+# lib/rspec_flaky/flaky_examples_collection.rb is requiring
+# `active_support/hash_with_indifferent_access`, and we install the `activesupport`
+# gem manually on the CI
+require 'rubygems'
+
 require_relative '../lib/rspec_flaky/report'
 
 report_file = ARGV.shift