2020-03-24 20:08:11 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2019-11-21 07:06:40 -05:00
|
|
|
require './spec/support/sidekiq_middleware'
|
2018-03-29 05:47:54 -04:00
|
|
|
require './spec/support/helpers/test_env'
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
# Usage:
|
|
|
|
#
|
|
|
|
# Simple invocation always creates a new project:
|
|
|
|
#
|
2020-11-27 19:09:43 -05:00
|
|
|
# FILTER=cycle_analytics SEED_VSA=1 bundle exec rake db:seed_fu
|
2020-03-24 20:08:11 -04:00
|
|
|
#
|
|
|
|
# Create more issues/MRs:
|
|
|
|
#
|
2020-11-27 19:09:43 -05:00
|
|
|
# VSA_ISSUE_COUNT=100 FILTER=cycle_analytics SEED_VSA=1 bundle exec rake db:seed_fu
|
2020-03-24 20:08:11 -04:00
|
|
|
#
|
|
|
|
# Run for an existing project
|
|
|
|
#
|
2020-11-27 19:09:43 -05:00
|
|
|
# VSA_SEED_PROJECT_ID=10 FILTER=cycle_analytics SEED_VSA=1 bundle exec rake db:seed_fu
|
2020-03-24 20:08:11 -04:00
|
|
|
|
2016-09-09 07:34:28 -04:00
|
|
|
class Gitlab::Seeder::CycleAnalytics
|
2020-03-24 20:08:11 -04:00
|
|
|
attr_reader :project, :issues, :merge_requests, :developers
|
|
|
|
|
2020-11-27 19:09:43 -05:00
|
|
|
FLAG = 'SEED_VSA'
|
|
|
|
PERF_TEST = 'VSA_PERF_TEST'
|
2020-03-24 20:08:11 -04:00
|
|
|
|
|
|
|
ISSUE_STAGE_MAX_DURATION_IN_HOURS = 72
|
|
|
|
PLAN_STAGE_MAX_DURATION_IN_HOURS = 48
|
|
|
|
CODE_STAGE_MAX_DURATION_IN_HOURS = 72
|
|
|
|
TEST_STAGE_MAX_DURATION_IN_HOURS = 5
|
|
|
|
REVIEW_STAGE_MAX_DURATION_IN_HOURS = 72
|
|
|
|
DEPLOYMENT_MAX_DURATION_IN_HOURS = 48
|
|
|
|
|
2020-03-25 05:08:11 -04:00
|
|
|
def self.seeder_based_on_env(project)
|
2020-03-24 20:08:11 -04:00
|
|
|
if ENV[FLAG]
|
|
|
|
self.new(project: project)
|
|
|
|
elsif ENV[PERF_TEST]
|
|
|
|
self.new(project: project, perf: true)
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def initialize(project: nil, perf: false)
|
|
|
|
@project = project || create_new_vsm_project
|
2020-11-27 19:09:43 -05:00
|
|
|
@issue_count = perf ? 1000 : ENV.fetch('VSA_ISSUE_COUNT', 5).to_i
|
2020-03-24 20:08:11 -04:00
|
|
|
@issues = []
|
|
|
|
@merge_requests = []
|
|
|
|
@developers = []
|
2016-09-15 10:12:12 -04:00
|
|
|
end
|
|
|
|
|
2016-09-09 07:34:28 -04:00
|
|
|
def seed!
|
2020-03-24 20:08:11 -04:00
|
|
|
create_developers!
|
|
|
|
create_issues!
|
|
|
|
|
|
|
|
seed_issue_stage!
|
|
|
|
seed_plan_stage!
|
|
|
|
seed_code_stage!
|
|
|
|
seed_test_stage!
|
|
|
|
seed_review_stage!
|
|
|
|
seed_staging_stage!
|
|
|
|
|
|
|
|
puts "Successfully seeded '#{project.full_path}' for Value Stream Management!"
|
|
|
|
puts "URL: #{Rails.application.routes.url_helpers.project_url(project)}"
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_issue_stage!
|
|
|
|
issues.each do |issue|
|
|
|
|
time = within_end_time(issue.created_at + rand(ISSUE_STAGE_MAX_DURATION_IN_HOURS).hours)
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
if issue.id.even?
|
|
|
|
issue.metrics.update!(first_associated_with_milestone_at: time)
|
2016-09-09 07:34:28 -04:00
|
|
|
else
|
2020-03-24 20:08:11 -04:00
|
|
|
issue.metrics.update!(first_added_to_board_at: time)
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_plan_stage!
|
|
|
|
issues.each do |issue|
|
|
|
|
plan_stage_start = issue.metrics.first_associated_with_milestone_at || issue.metrics.first_added_to_board_at
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
first_mentioned_in_commit_at = within_end_time(plan_stage_start + rand(PLAN_STAGE_MAX_DURATION_IN_HOURS).hours)
|
|
|
|
issue.metrics.update!(first_mentioned_in_commit_at: first_mentioned_in_commit_at)
|
|
|
|
end
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_code_stage!
|
|
|
|
issues.each do |issue|
|
|
|
|
merge_request = FactoryBot.create(
|
|
|
|
:merge_request,
|
|
|
|
target_project: project,
|
|
|
|
source_project: project,
|
|
|
|
source_branch: "#{issue.iid}-feature-branch",
|
|
|
|
target_branch: 'master',
|
|
|
|
author: developers.sample,
|
|
|
|
created_at: within_end_time(issue.metrics.first_mentioned_in_commit_at + rand(CODE_STAGE_MAX_DURATION_IN_HOURS).hours)
|
|
|
|
)
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
@merge_requests << merge_request
|
Improve performance of the cycle analytics page.
1. These changes bring down page load time for 100 issues from more than
a minute to about 1.5 seconds.
2. This entire commit is composed of these types of performance
enhancements:
- Cache relevant data in `IssueMetrics` wherever possible.
- Cache relevant data in `MergeRequestMetrics` wherever possible.
- Preload metrics
3. Given these improvements, we now only need to make 4 SQL calls:
- Load all issues
- Load all merge requests
- Load all metrics for the issues
- Load all metrics for the merge requests
4. A list of all the data points that are now being pre-calculated:
a. The first time an issue is mentioned in a commit
- In `GitPushService`, find all issues mentioned by the given commit
using `ReferenceExtractor`. Set the `first_mentioned_in_commit_at`
flag for each of them.
- There seems to be a (pre-existing) bug here - files (and
therefore commits) created using the Web CI don't have
cross-references created, and issues are not closed even when
the commit title is "Fixes #xx".
b. The first time a merge request is deployed to production
When a `Deployment` is created, find all merge requests that
were merged in before the deployment, and set the
`first_deployed_to_production_at` flag for each of them.
c. The start / end time for a merge request pipeline
Hook into the `Pipeline` state machine. When the `status` moves to
`running`, find the merge requests whose tip commit matches the
pipeline, and record the `latest_build_started_at` time for each
of them. When the `status` moves to `success`, record the
`latest_build_finished_at` time.
d. The merge requests that close an issue
- This was a big cause of the performance problems we were having
with Cycle Analytics. We need to use `ReferenceExtractor` to make
this calculation, which is slow when we have to run it on a large
number of merge requests.
- When a merge request is created, updated, or refreshed, find the
issues it closes, and create an instance of
`MergeRequestsClosingIssues`, which acts as a join model between
merge requests and issues.
- If a `MergeRequestsClosingIssues` instance links a merge request
and an issue, that issue closes that merge request.
5. The `Queries` module was changed into a class, so we can cache the
results of `issues` and `merge_requests_closing_issues` across
various cycle analytics stages.
6. The code added in this commit is untested. Tests will be added in the
next commit.
2016-09-15 04:59:36 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
MergeRequestsClosingIssues.create!(issue: issue, merge_request: merge_request)
|
|
|
|
end
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_test_stage!
|
|
|
|
merge_requests.each do |merge_request|
|
|
|
|
pipeline = FactoryBot.create(:ci_pipeline, :success, project: project)
|
|
|
|
build = FactoryBot.create(:ci_build, pipeline: pipeline, project: project, user: developers.sample)
|
|
|
|
|
2020-06-03 20:08:17 -04:00
|
|
|
# Required because seeds run in a transaction and these are now
|
|
|
|
# created in an `after_commit` hook.
|
|
|
|
merge_request.ensure_metrics
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
merge_request.metrics.update!(
|
|
|
|
latest_build_started_at: merge_request.created_at,
|
|
|
|
latest_build_finished_at: within_end_time(merge_request.created_at + TEST_STAGE_MAX_DURATION_IN_HOURS.hours),
|
|
|
|
pipeline_id: build.commit_id
|
|
|
|
)
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_review_stage!
|
|
|
|
merge_requests.each do |merge_request|
|
|
|
|
merge_request.metrics.update!(merged_at: within_end_time(merge_request.created_at + REVIEW_STAGE_MAX_DURATION_IN_HOURS.hours))
|
|
|
|
end
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def seed_staging_stage!
|
|
|
|
merge_requests.each do |merge_request|
|
|
|
|
merge_request.metrics.update!(first_deployed_to_production_at: within_end_time(merge_request.metrics.merged_at + DEPLOYMENT_MAX_DURATION_IN_HOURS.hours))
|
|
|
|
end
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def create_issues!
|
|
|
|
@issue_count.times do
|
|
|
|
Timecop.travel start_time + rand(5).days do
|
|
|
|
title = "#{FFaker::Product.brand}-#{suffix}"
|
|
|
|
@issues << Issue.create!(project: project, title: title, author: developers.sample)
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def create_developers!
|
|
|
|
5.times do |i|
|
|
|
|
user = FactoryBot.create(
|
|
|
|
:user,
|
|
|
|
name: "VSM User#{i}",
|
|
|
|
username: "vsm-user-#{i}-#{suffix}",
|
|
|
|
email: "vsm-user-#{i}@#{suffix}.com"
|
|
|
|
)
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2021-08-02 02:10:20 -04:00
|
|
|
project.group&.add_developer(user)
|
2020-03-24 20:08:11 -04:00
|
|
|
project.add_developer(user)
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
@developers << user
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def create_new_vsm_project
|
|
|
|
project = FactoryBot.create(
|
|
|
|
:project,
|
|
|
|
name: "Value Stream Management Project #{suffix}",
|
|
|
|
path: "vsmp-#{suffix}",
|
|
|
|
creator: admin,
|
|
|
|
namespace: FactoryBot.create(
|
|
|
|
:group,
|
2020-03-26 20:08:09 -04:00
|
|
|
name: "Value Stream Management Group #{suffix}",
|
2020-03-24 20:08:11 -04:00
|
|
|
path: "vsmg-#{suffix}"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
project.create_repository
|
|
|
|
project
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def admin
|
|
|
|
@admin ||= User.admins.first
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def suffix
|
|
|
|
@suffix ||= Time.now.to_i
|
|
|
|
end
|
2017-11-16 16:26:12 -05:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def start_time
|
|
|
|
@start_time ||= 25.days.ago
|
|
|
|
end
|
2016-09-09 07:34:28 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def end_time
|
|
|
|
@end_time ||= Time.now
|
|
|
|
end
|
2017-06-01 14:46:34 -04:00
|
|
|
|
2020-03-24 20:08:11 -04:00
|
|
|
def within_end_time(time)
|
|
|
|
[time, end_time].min
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
Gitlab::Seeder.quiet do
|
2020-11-27 19:09:43 -05:00
|
|
|
project_id = ENV['VSA_SEED_PROJECT_ID']
|
2020-03-24 20:08:11 -04:00
|
|
|
project = Project.find(project_id) if project_id
|
|
|
|
|
2020-03-25 05:08:11 -04:00
|
|
|
seeder = Gitlab::Seeder::CycleAnalytics.seeder_based_on_env(project)
|
2020-03-24 20:08:11 -04:00
|
|
|
|
|
|
|
if seeder
|
2016-09-09 07:34:28 -04:00
|
|
|
seeder.seed!
|
2016-09-14 10:10:31 -04:00
|
|
|
else
|
2020-03-24 20:08:11 -04:00
|
|
|
puts "Skipped. Use the `#{Gitlab::Seeder::CycleAnalytics::FLAG}` environment variable to enable."
|
2016-09-09 07:34:28 -04:00
|
|
|
end
|
|
|
|
end
|