2018-08-03 03:15:25 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-09-21 04:34:12 -04:00
|
|
|
module Ci
|
2019-03-28 09:17:42 -04:00
|
|
|
class JobArtifact < ApplicationRecord
|
2017-12-08 04:09:06 -05:00
|
|
|
include AfterCommitQueue
|
2018-02-21 11:43:21 -05:00
|
|
|
include ObjectStorage::BackgroundMove
|
2019-04-19 05:37:14 -04:00
|
|
|
include UpdateProjectStatistics
|
2019-09-18 17:06:34 -04:00
|
|
|
include Sortable
|
2017-09-21 04:34:12 -04:00
|
|
|
extend Gitlab::Ci::Model
|
|
|
|
|
2018-08-02 02:05:07 -04:00
|
|
|
NotSupportedAdapterError = Class.new(StandardError)
|
|
|
|
|
2018-07-27 01:04:35 -04:00
|
|
|
TEST_REPORT_FILE_TYPES = %w[junit].freeze
|
2020-03-17 14:09:44 -04:00
|
|
|
COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze
|
2020-04-22 23:09:51 -04:00
|
|
|
ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze
|
2018-09-27 17:15:08 -04:00
|
|
|
NON_ERASABLE_FILE_TYPES = %w[trace].freeze
|
2020-04-21 11:21:10 -04:00
|
|
|
TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
|
2018-09-27 17:15:08 -04:00
|
|
|
DEFAULT_FILE_NAMES = {
|
|
|
|
archive: nil,
|
|
|
|
metadata: nil,
|
|
|
|
trace: nil,
|
2020-01-15 16:08:48 -05:00
|
|
|
metrics_referee: nil,
|
|
|
|
network_referee: nil,
|
2018-09-27 17:15:08 -04:00
|
|
|
junit: 'junit.xml',
|
2020-04-22 23:09:51 -04:00
|
|
|
accessibility: 'gl-accessibility.json',
|
2018-11-05 11:32:03 -05:00
|
|
|
codequality: 'gl-code-quality-report.json',
|
2018-09-27 17:15:08 -04:00
|
|
|
sast: 'gl-sast-report.json',
|
|
|
|
dependency_scanning: 'gl-dependency-scanning-report.json',
|
|
|
|
container_scanning: 'gl-container-scanning-report.json',
|
2018-10-07 04:22:40 -04:00
|
|
|
dast: 'gl-dast-report.json',
|
|
|
|
license_management: 'gl-license-management-report.json',
|
2020-01-09 13:07:52 -05:00
|
|
|
license_scanning: 'gl-license-scanning-report.json',
|
2019-04-16 07:06:52 -04:00
|
|
|
performance: 'performance.json',
|
2020-01-28 13:08:35 -05:00
|
|
|
metrics: 'metrics.txt',
|
2020-03-13 17:09:38 -04:00
|
|
|
lsif: 'lsif.json',
|
2020-03-17 14:09:44 -04:00
|
|
|
dotenv: '.env',
|
2020-03-24 14:07:55 -04:00
|
|
|
cobertura: 'cobertura-coverage.xml',
|
2020-05-11 23:09:31 -04:00
|
|
|
terraform: 'tfplan.json',
|
|
|
|
cluster_applications: 'gl-cluster-applications.json'
|
2018-09-27 17:15:08 -04:00
|
|
|
}.freeze
|
|
|
|
|
2019-05-20 18:56:19 -04:00
|
|
|
INTERNAL_TYPES = {
|
2018-09-27 17:15:08 -04:00
|
|
|
archive: :zip,
|
|
|
|
metadata: :gzip,
|
2019-05-20 18:56:19 -04:00
|
|
|
trace: :raw
|
|
|
|
}.freeze
|
|
|
|
|
|
|
|
REPORT_TYPES = {
|
2018-09-27 17:15:08 -04:00
|
|
|
junit: :gzip,
|
2019-04-16 07:06:52 -04:00
|
|
|
metrics: :gzip,
|
2020-01-15 16:08:48 -05:00
|
|
|
metrics_referee: :gzip,
|
|
|
|
network_referee: :gzip,
|
2020-02-18 07:09:15 -05:00
|
|
|
lsif: :gzip,
|
2020-03-13 17:09:38 -04:00
|
|
|
dotenv: :gzip,
|
2020-03-17 14:09:44 -04:00
|
|
|
cobertura: :gzip,
|
2020-05-11 23:09:31 -04:00
|
|
|
cluster_applications: :gzip,
|
2018-10-15 06:17:21 -04:00
|
|
|
|
|
|
|
# All these file formats use `raw` as we need to store them uncompressed
|
|
|
|
# for Frontend to fetch the files and do analysis
|
|
|
|
# When they will be only used by backend, they can be `gzipped`.
|
2020-04-22 23:09:51 -04:00
|
|
|
accessibility: :raw,
|
2018-10-15 06:17:21 -04:00
|
|
|
codequality: :raw,
|
|
|
|
sast: :raw,
|
|
|
|
dependency_scanning: :raw,
|
|
|
|
container_scanning: :raw,
|
2018-10-07 04:22:40 -04:00
|
|
|
dast: :raw,
|
|
|
|
license_management: :raw,
|
2020-01-08 10:08:01 -05:00
|
|
|
license_scanning: :raw,
|
2020-03-24 14:07:55 -04:00
|
|
|
performance: :raw,
|
|
|
|
terraform: :raw
|
2018-09-27 17:15:08 -04:00
|
|
|
}.freeze
|
2018-07-27 01:04:35 -04:00
|
|
|
|
2019-05-20 18:56:19 -04:00
|
|
|
TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze
|
|
|
|
|
2020-04-27 17:10:10 -04:00
|
|
|
# This is required since we cannot add a default to the database
|
|
|
|
# https://gitlab.com/gitlab-org/gitlab/-/issues/215418
|
|
|
|
attribute :locked, :boolean, default: false
|
|
|
|
|
2017-09-21 04:34:12 -04:00
|
|
|
belongs_to :project
|
2017-11-02 14:38:25 -04:00
|
|
|
belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
|
2017-09-21 04:34:12 -04:00
|
|
|
|
2018-04-13 04:20:07 -04:00
|
|
|
mount_uploader :file, JobArtifactUploader
|
|
|
|
|
2018-07-27 01:04:35 -04:00
|
|
|
validates :file_format, presence: true, unless: :trace?, on: :create
|
|
|
|
validate :valid_file_format?, unless: :trace?, on: :create
|
2020-04-14 11:09:44 -04:00
|
|
|
before_save :set_size, if: :file_changed?
|
2017-09-21 04:34:12 -04:00
|
|
|
|
2020-04-14 11:09:44 -04:00
|
|
|
update_project_statistics project_statistics_name: :build_artifacts_size
|
|
|
|
|
2020-04-15 14:09:36 -04:00
|
|
|
after_save :update_file_store, if: :saved_change_to_file?
|
|
|
|
|
2018-04-13 04:20:07 -04:00
|
|
|
scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
|
2019-09-20 08:05:52 -04:00
|
|
|
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
|
2020-02-23 22:09:05 -05:00
|
|
|
scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
|
2020-04-27 17:10:10 -04:00
|
|
|
scope :for_ref, ->(ref, project_id) { joins(job: :pipeline).where(ci_pipelines: { ref: ref, project_id: project_id }) }
|
2020-03-24 14:07:55 -04:00
|
|
|
scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) }
|
2017-12-08 04:09:06 -05:00
|
|
|
|
2018-09-27 17:15:08 -04:00
|
|
|
scope :with_file_types, -> (file_types) do
|
|
|
|
types = self.file_types.select { |file_type| file_types.include?(file_type) }.values
|
|
|
|
|
|
|
|
where(file_type: types)
|
|
|
|
end
|
|
|
|
|
2019-05-20 18:56:19 -04:00
|
|
|
scope :with_reports, -> do
|
|
|
|
with_file_types(REPORT_TYPES.keys.map(&:to_s))
|
2019-05-01 17:31:04 -04:00
|
|
|
end
|
|
|
|
|
2018-07-27 01:04:35 -04:00
|
|
|
scope :test_reports, -> do
|
2018-09-27 17:15:08 -04:00
|
|
|
with_file_types(TEST_REPORT_FILE_TYPES)
|
|
|
|
end
|
|
|
|
|
2020-04-22 23:09:51 -04:00
|
|
|
scope :accessibility_reports, -> do
|
|
|
|
with_file_types(ACCESSIBILITY_REPORT_FILE_TYPES)
|
|
|
|
end
|
|
|
|
|
2020-03-17 14:09:44 -04:00
|
|
|
scope :coverage_reports, -> do
|
|
|
|
with_file_types(COVERAGE_REPORT_FILE_TYPES)
|
|
|
|
end
|
|
|
|
|
2020-04-21 11:21:10 -04:00
|
|
|
scope :terraform_reports, -> do
|
|
|
|
with_file_types(TERRAFORM_REPORT_FILE_TYPES)
|
|
|
|
end
|
|
|
|
|
2018-09-27 17:15:08 -04:00
|
|
|
scope :erasable, -> do
|
|
|
|
types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
|
2018-07-27 01:04:35 -04:00
|
|
|
|
|
|
|
where(file_type: types)
|
|
|
|
end
|
|
|
|
|
2019-01-17 01:06:37 -05:00
|
|
|
scope :expired, -> (limit) { where('expire_at < ?', Time.now).limit(limit) }
|
2020-04-27 17:10:10 -04:00
|
|
|
scope :locked, -> { where(locked: true) }
|
|
|
|
scope :unlocked, -> { where(locked: [false, nil]) }
|
2019-01-17 01:06:37 -05:00
|
|
|
|
2019-08-28 10:26:42 -04:00
|
|
|
scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }
|
|
|
|
|
2018-07-18 17:46:56 -04:00
|
|
|
delegate :filename, :exists?, :open, to: :file
|
2018-02-06 09:18:32 -05:00
|
|
|
|
2017-09-21 04:34:12 -04:00
|
|
|
enum file_type: {
|
|
|
|
archive: 1,
|
2018-02-06 09:18:32 -05:00
|
|
|
metadata: 2,
|
2018-07-27 01:04:35 -04:00
|
|
|
trace: 3,
|
2018-09-27 17:15:08 -04:00
|
|
|
junit: 4,
|
|
|
|
sast: 5, ## EE-specific
|
|
|
|
dependency_scanning: 6, ## EE-specific
|
|
|
|
container_scanning: 7, ## EE-specific
|
2018-10-02 13:01:26 -04:00
|
|
|
dast: 8, ## EE-specific
|
2018-10-07 04:22:40 -04:00
|
|
|
codequality: 9, ## EE-specific
|
|
|
|
license_management: 10, ## EE-specific
|
2020-01-08 10:08:01 -05:00
|
|
|
license_scanning: 101, ## EE-specific till 13.0
|
2019-04-16 07:06:52 -04:00
|
|
|
performance: 11, ## EE-specific
|
2020-01-15 16:08:48 -05:00
|
|
|
metrics: 12, ## EE-specific
|
|
|
|
metrics_referee: 13, ## runner referees
|
2020-01-28 13:08:35 -05:00
|
|
|
network_referee: 14, ## runner referees
|
2020-03-13 17:09:38 -04:00
|
|
|
lsif: 15, # LSIF data for code navigation
|
2020-03-17 14:09:44 -04:00
|
|
|
dotenv: 16,
|
2020-03-24 14:07:55 -04:00
|
|
|
cobertura: 17,
|
2020-04-22 23:09:51 -04:00
|
|
|
terraform: 18, # Transformed json
|
2020-05-11 23:09:31 -04:00
|
|
|
accessibility: 19,
|
|
|
|
cluster_applications: 20
|
2017-09-21 04:34:12 -04:00
|
|
|
}
|
|
|
|
|
2018-07-27 01:04:35 -04:00
|
|
|
enum file_format: {
|
|
|
|
raw: 1,
|
|
|
|
zip: 2,
|
|
|
|
gzip: 3
|
2019-05-13 00:42:06 -04:00
|
|
|
}, _suffix: true
|
2018-07-27 01:04:35 -04:00
|
|
|
|
2018-08-17 00:33:15 -04:00
|
|
|
# `file_location` indicates where actual files are stored.
|
|
|
|
# Ideally, actual files should be stored in the same directory, and use the same
|
|
|
|
# convention to generate its path. However, sometimes we can't do so due to backward-compatibility.
|
|
|
|
#
|
|
|
|
# legacy_path ... The actual file is stored at a path consists of a timestamp
|
|
|
|
# and raw project/model IDs. Those rows were migrated from
|
|
|
|
# `ci_builds.artifacts_file` and `ci_builds.artifacts_metadata`
|
|
|
|
# hashed_path ... The actual file is stored at a path consists of a SHA2 based on the project ID.
|
|
|
|
# This is the default value.
|
|
|
|
enum file_location: {
|
|
|
|
legacy_path: 1,
|
|
|
|
hashed_path: 2
|
|
|
|
}
|
|
|
|
|
2018-08-02 02:05:07 -04:00
|
|
|
FILE_FORMAT_ADAPTERS = {
|
2018-10-15 06:17:21 -04:00
|
|
|
gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream,
|
|
|
|
raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream
|
2018-08-02 02:05:07 -04:00
|
|
|
}.freeze
|
|
|
|
|
2018-07-27 01:04:35 -04:00
|
|
|
def valid_file_format?
|
|
|
|
unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
|
2020-03-03 04:07:54 -05:00
|
|
|
errors.add(:base, _('Invalid file format with specified file type'))
|
2018-07-27 01:04:35 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-04-03 12:47:33 -04:00
|
|
|
def update_file_store
|
2018-04-13 04:20:07 -04:00
|
|
|
# The file.object_store is set during `uploader.store!`
|
|
|
|
# which happens after object is inserted/updated
|
|
|
|
self.update_column(:file_store, file.object_store)
|
2018-04-03 12:47:33 -04:00
|
|
|
end
|
|
|
|
|
2019-09-18 17:06:34 -04:00
|
|
|
def self.total_size
|
|
|
|
self.sum(:size)
|
|
|
|
end
|
|
|
|
|
2017-09-21 04:34:12 -04:00
|
|
|
def self.artifacts_size_for(project)
|
|
|
|
self.where(project: project).sum(:size)
|
|
|
|
end
|
|
|
|
|
2018-02-21 11:43:21 -05:00
|
|
|
def local_store?
|
|
|
|
[nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
|
|
|
|
end
|
|
|
|
|
2018-08-17 00:33:15 -04:00
|
|
|
def hashed_path?
|
2018-09-05 12:12:46 -04:00
|
|
|
return true if trace? # ArchiveLegacyTraces background migration might not have `file_location` column
|
|
|
|
|
|
|
|
super || self.file_location.nil?
|
2018-08-17 00:33:15 -04:00
|
|
|
end
|
|
|
|
|
2017-12-03 06:02:11 -05:00
|
|
|
def expire_in
|
|
|
|
expire_at - Time.now if expire_at
|
|
|
|
end
|
|
|
|
|
|
|
|
def expire_in=(value)
|
|
|
|
self.expire_at =
|
|
|
|
if value
|
|
|
|
ChronicDuration.parse(value)&.seconds&.from_now
|
|
|
|
end
|
|
|
|
end
|
2018-03-20 19:03:50 -04:00
|
|
|
|
2018-08-02 02:05:07 -04:00
|
|
|
def each_blob(&blk)
|
|
|
|
unless file_format_adapter_class
|
|
|
|
raise NotSupportedAdapterError, 'This file format requires a dedicated adapter'
|
|
|
|
end
|
|
|
|
|
|
|
|
file.open do |stream|
|
|
|
|
file_format_adapter_class.new(stream).each_blob(&blk)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-07-18 05:22:46 -04:00
|
|
|
def self.archived_trace_exists_for?(job_id)
|
|
|
|
where(job_id: job_id).trace.take&.file&.file&.exists?
|
|
|
|
end
|
|
|
|
|
2018-03-20 19:03:50 -04:00
|
|
|
private
|
|
|
|
|
2018-08-02 02:05:07 -04:00
|
|
|
def file_format_adapter_class
|
|
|
|
FILE_FORMAT_ADAPTERS[file_format.to_sym]
|
|
|
|
end
|
|
|
|
|
2018-03-20 19:03:50 -04:00
|
|
|
def set_size
|
|
|
|
self.size = file.size
|
|
|
|
end
|
|
|
|
|
|
|
|
def project_destroyed?
|
|
|
|
# Use job.project to avoid extra DB query for project
|
|
|
|
job.project.pending_delete?
|
|
|
|
end
|
2017-09-21 04:34:12 -04:00
|
|
|
end
|
|
|
|
end
|
2019-09-13 09:26:31 -04:00
|
|
|
|
|
|
|
Ci::JobArtifact.prepend_if_ee('EE::Ci::JobArtifact')
|