0103d5be96
These are data columns that store runtime configuration of build needed to execute it on runner and within pipeline. The definition of this data is that once used, and when no longer needed (due to retry capability) they can be freely removed. They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking support for json datatype on old enough version).
282 lines
9.2 KiB
Ruby
282 lines
9.2 KiB
Ruby
require './spec/support/sidekiq'
|
|
|
|
class Gitlab::Seeder::Pipelines
|
|
STAGES = %w[build test security deploy notify]
|
|
BUILDS = [
|
|
# build stage
|
|
{ name: 'build:linux', stage: 'build', status: :success,
|
|
queued_at: 10.hour.ago, started_at: 9.hour.ago, finished_at: 8.hour.ago },
|
|
{ name: 'build:osx', stage: 'build', status: :success,
|
|
queued_at: 10.hour.ago, started_at: 10.hour.ago, finished_at: 9.hour.ago },
|
|
|
|
# test stage
|
|
{ name: 'rspec:linux 0 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:linux 1 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:linux 2 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:windows 0 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:windows 1 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:windows 2 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:windows 2 3', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'rspec:osx', stage: 'test', status_event: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'spinach:linux', stage: 'test', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'spinach:osx', stage: 'test', status: :failed, allow_failure: true,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
|
|
# security stage
|
|
{ name: 'dast', stage: 'security', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'sast', stage: 'security', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'dependency_scanning', stage: 'security', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
{ name: 'container_scanning', stage: 'security', status: :success,
|
|
queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago },
|
|
|
|
# deploy stage
|
|
{ name: 'staging', stage: 'deploy', environment: 'staging', status_event: :success,
|
|
options: { environment: { action: 'start', on_stop: 'stop staging' } },
|
|
queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago },
|
|
{ name: 'stop staging', stage: 'deploy', environment: 'staging',
|
|
when: 'manual', status: :skipped },
|
|
{ name: 'production', stage: 'deploy', environment: 'production',
|
|
when: 'manual', status: :skipped },
|
|
|
|
# notify stage
|
|
{ name: 'slack', stage: 'notify', when: 'manual', status: :success },
|
|
]
|
|
EXTERNAL_JOBS = [
|
|
{ name: 'jenkins', stage: 'test', status: :success,
|
|
queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago },
|
|
]
|
|
|
|
def initialize(project)
|
|
@project = project
|
|
end
|
|
|
|
def seed!
|
|
pipelines.each do |pipeline|
|
|
BUILDS.each { |opts| build_create!(pipeline, opts) }
|
|
EXTERNAL_JOBS.each { |opts| commit_status_create!(pipeline, opts) }
|
|
pipeline.update_duration
|
|
pipeline.update_status
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def pipelines
|
|
create_master_pipelines + create_merge_request_pipelines
|
|
end
|
|
|
|
def create_master_pipelines
|
|
@project.repository.commits('master', limit: 4).map do |commit|
|
|
create_pipeline!(@project, 'master', commit)
|
|
end
|
|
rescue
|
|
[]
|
|
end
|
|
|
|
def create_merge_request_pipelines
|
|
pipelines = @project.merge_requests.first(3).map do |merge_request|
|
|
project = merge_request.source_project
|
|
branch = merge_request.source_branch
|
|
|
|
merge_request.commits.last(4).map do |commit|
|
|
create_pipeline!(project, branch, commit).tap do |pipeline|
|
|
merge_request.update!(head_pipeline_id: pipeline.id)
|
|
end
|
|
end
|
|
end
|
|
|
|
pipelines.flatten
|
|
rescue
|
|
[]
|
|
end
|
|
|
|
def create_pipeline!(project, ref, commit)
|
|
project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
|
|
end
|
|
|
|
def build_create!(pipeline, opts = {})
|
|
attributes = job_attributes(pipeline, opts)
|
|
|
|
attributes[:options] ||= {}
|
|
attributes[:options][:script] = 'build command'
|
|
|
|
Ci::Build.create!(attributes).tap do |build|
|
|
# We need to set build trace and artifacts after saving a build
|
|
# (id required), that is why we need `#tap` method instead of passing
|
|
# block directly to `Ci::Build#create!`.
|
|
|
|
setup_artifacts(build)
|
|
setup_test_reports(build)
|
|
if build.ref == build.project.default_branch
|
|
setup_security_reports_file(build)
|
|
else
|
|
setup_security_reports_legacy_archive(build)
|
|
end
|
|
setup_build_log(build)
|
|
|
|
build.project.environments.
|
|
find_or_create_by(name: build.expanded_environment_name)
|
|
|
|
build.save!
|
|
end
|
|
end
|
|
|
|
def setup_artifacts(build)
|
|
return unless build.stage == "build"
|
|
|
|
artifacts_cache_file(artifacts_archive_path) do |file|
|
|
build.job_artifacts.build(project: build.project, file_type: :archive, file_format: :zip, file: file)
|
|
end
|
|
|
|
artifacts_cache_file(artifacts_metadata_path) do |file|
|
|
build.job_artifacts.build(project: build.project, file_type: :metadata, file_format: :gzip, file: file)
|
|
end
|
|
end
|
|
|
|
def setup_test_reports(build)
|
|
return unless build.stage == "test" && build.name == "rspec:osx"
|
|
|
|
if build.ref == build.project.default_branch
|
|
artifacts_cache_file(test_reports_pass_path) do |file|
|
|
build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file)
|
|
end
|
|
else
|
|
artifacts_cache_file(test_reports_failed_path) do |file|
|
|
build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file)
|
|
end
|
|
end
|
|
end
|
|
|
|
def setup_security_reports_file(build)
|
|
return unless build.stage == "security"
|
|
|
|
# we have two sources: master and feature-branch
|
|
branch_name = build.ref == build.project.default_branch ?
|
|
'master' : 'feature-branch'
|
|
|
|
artifacts_cache_file(security_reports_path(branch_name, build.name)) do |file|
|
|
build.job_artifacts.build(
|
|
project: build.project,
|
|
file_type: build.name,
|
|
file_format: :raw,
|
|
file: file)
|
|
end
|
|
end
|
|
|
|
def setup_security_reports_legacy_archive(build)
|
|
return unless build.stage == "security"
|
|
|
|
# we have two sources: master and feature-branch
|
|
branch_name = build.ref == build.project.default_branch ?
|
|
'master' : 'feature-branch'
|
|
|
|
artifacts_cache_file(security_reports_archive_path(branch_name)) do |file|
|
|
build.job_artifacts.build(
|
|
project: build.project,
|
|
file_type: :archive,
|
|
file_format: :zip,
|
|
file: file)
|
|
end
|
|
|
|
# assign dummy metadata
|
|
artifacts_cache_file(artifacts_metadata_path) do |file|
|
|
build.job_artifacts.build(
|
|
project: build.project,
|
|
file_type: :metadata,
|
|
file_format: :gzip,
|
|
file: file)
|
|
end
|
|
|
|
build.options = {
|
|
artifacts: {
|
|
paths: [
|
|
Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(build.name.to_sym)
|
|
]
|
|
}
|
|
}
|
|
end
|
|
|
|
def setup_build_log(build)
|
|
if %w(running success failed).include?(build.status)
|
|
build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n"))
|
|
end
|
|
end
|
|
|
|
def commit_status_create!(pipeline, opts = {})
|
|
attributes = job_attributes(pipeline, opts)
|
|
|
|
GenericCommitStatus.create!(attributes)
|
|
end
|
|
|
|
def job_attributes(pipeline, opts)
|
|
{ name: 'test build', stage: 'test', stage_idx: stage_index(opts[:stage]),
|
|
ref: pipeline.ref, tag: false, user: build_user, project: @project, pipeline: pipeline,
|
|
created_at: Time.now, updated_at: Time.now
|
|
}.merge(opts)
|
|
end
|
|
|
|
def build_user
|
|
@project.team.users.sample
|
|
end
|
|
|
|
def build_status
|
|
Ci::Build::AVAILABLE_STATUSES.sample
|
|
end
|
|
|
|
def stage_index(stage)
|
|
STAGES.index(stage) || 0
|
|
end
|
|
|
|
def artifacts_archive_path
|
|
Rails.root + 'spec/fixtures/ci_build_artifacts.zip'
|
|
end
|
|
|
|
def artifacts_metadata_path
|
|
Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz'
|
|
end
|
|
|
|
def test_reports_pass_path
|
|
Rails.root + 'spec/fixtures/junit/junit_ant.xml.gz'
|
|
end
|
|
|
|
def test_reports_failed_path
|
|
Rails.root + 'spec/fixtures/junit/junit.xml.gz'
|
|
end
|
|
|
|
def security_reports_archive_path(branch)
|
|
Rails.root.join('spec', 'fixtures', 'security-reports', branch + '.zip')
|
|
end
|
|
|
|
def security_reports_path(branch, name)
|
|
file_name = Ci::JobArtifact::DEFAULT_FILE_NAMES.fetch(name.to_sym)
|
|
Rails.root.join('spec', 'fixtures', 'security-reports', branch, file_name)
|
|
end
|
|
|
|
def artifacts_cache_file(file_path)
|
|
file = Tempfile.new("artifacts")
|
|
file.close
|
|
|
|
FileUtils.copy(file_path, file.path)
|
|
|
|
yield(UploadedFile.new(file.path, filename: File.basename(file_path)))
|
|
end
|
|
end
|
|
|
|
Gitlab::Seeder.quiet do
|
|
Project.all.sample(5).each do |project|
|
|
project_builds = Gitlab::Seeder::Pipelines.new(project)
|
|
project_builds.seed!
|
|
end
|
|
end
|