2018-10-26 00:12:43 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
module Gitlab
|
|
|
|
module Ci
|
|
|
|
class Trace
|
2018-11-23 11:25:11 -05:00
|
|
|
include ::Gitlab::ExclusiveLeaseHelpers
|
2021-09-06 14:11:17 -04:00
|
|
|
include ::Gitlab::Utils::StrongMemoize
|
2019-10-08 20:06:06 -04:00
|
|
|
include Checksummable
|
2018-05-31 03:11:53 -04:00
|
|
|
|
2019-06-25 22:07:44 -04:00
|
|
|
LOCK_TTL = 10.minutes
|
2018-11-23 11:25:11 -05:00
|
|
|
LOCK_RETRIES = 2
|
|
|
|
LOCK_SLEEP = 0.001.seconds
|
2020-01-13 04:08:03 -05:00
|
|
|
WATCH_FLAG_TTL = 10.seconds
|
|
|
|
|
2021-03-23 08:09:33 -04:00
|
|
|
UPDATE_FREQUENCY_DEFAULT = 60.seconds
|
2020-01-13 04:08:03 -05:00
|
|
|
UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds
|
2018-05-31 03:11:53 -04:00
|
|
|
|
2021-06-04 11:10:25 -04:00
|
|
|
LOAD_BALANCING_STICKING_NAMESPACE = 'ci/build/trace'
|
|
|
|
|
2018-03-02 09:30:31 -05:00
|
|
|
ArchiveError = Class.new(StandardError)
|
2018-07-06 02:19:01 -04:00
|
|
|
AlreadyArchivedError = Class.new(StandardError)
|
2020-09-28 11:09:44 -04:00
|
|
|
LockedError = Class.new(StandardError)
|
2018-03-02 09:30:31 -05:00
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
attr_reader :job
|
|
|
|
|
|
|
|
delegate :old_trace, to: :job
|
2021-09-06 14:11:17 -04:00
|
|
|
delegate :can_attempt_archival_now?, :increment_archival_attempts!,
|
|
|
|
:archival_attempts_message, to: :trace_metadata
|
2017-04-06 12:20:27 -04:00
|
|
|
|
|
|
|
def initialize(job)
|
|
|
|
@job = job
|
|
|
|
end
|
|
|
|
|
|
|
|
def html(last_lines: nil)
|
|
|
|
read do |stream|
|
|
|
|
stream.html(last_lines: last_lines)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def raw(last_lines: nil)
|
|
|
|
read do |stream|
|
|
|
|
stream.raw(last_lines: last_lines)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def extract_coverage(regex)
|
|
|
|
read do |stream|
|
|
|
|
stream.extract_coverage(regex)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-09-25 12:54:08 -04:00
|
|
|
def extract_sections
|
|
|
|
read do |stream|
|
|
|
|
stream.extract_sections
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
def set(data)
|
2018-04-05 03:16:01 -04:00
|
|
|
write('w+b') do |stream|
|
2017-04-06 12:20:27 -04:00
|
|
|
data = job.hide_secrets(data)
|
|
|
|
stream.set(data)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def append(data, offset)
|
2018-04-18 02:19:53 -04:00
|
|
|
write('a+b') do |stream|
|
2017-04-06 12:20:27 -04:00
|
|
|
current_length = stream.size
|
2018-05-06 13:54:41 -04:00
|
|
|
break current_length unless current_length == offset
|
2017-04-06 12:20:27 -04:00
|
|
|
|
|
|
|
data = job.hide_secrets(data)
|
|
|
|
stream.append(data, offset)
|
|
|
|
stream.size
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def exist?
|
2019-07-18 05:22:46 -04:00
|
|
|
archived_trace_exist? || live_trace_exist?
|
|
|
|
end
|
|
|
|
|
|
|
|
def archived_trace_exist?
|
|
|
|
trace_artifact&.exists?
|
|
|
|
end
|
|
|
|
|
|
|
|
def live_trace_exist?
|
|
|
|
job.trace_chunks.any? || current_path.present? || old_trace.present?
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
2020-10-13 11:08:53 -04:00
|
|
|
def read(&block)
|
2020-08-24 14:10:19 -04:00
|
|
|
read_stream(&block)
|
2020-10-13 11:08:53 -04:00
|
|
|
rescue Errno::ENOENT, ChunkedIO::FailedToGetChunkError
|
2020-08-24 14:10:19 -04:00
|
|
|
job.reset
|
|
|
|
read_stream(&block)
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
2018-11-23 11:25:11 -05:00
|
|
|
def write(mode, &blk)
|
|
|
|
in_write_lock do
|
|
|
|
unsafe_write!(mode, &blk)
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-04-06 08:09:21 -04:00
|
|
|
def erase_trace_chunks!
|
|
|
|
job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
def erase!
|
2018-07-03 01:00:25 -04:00
|
|
|
##
|
2018-07-04 01:37:33 -04:00
|
|
|
# Erase the archived trace
|
2018-07-03 01:00:25 -04:00
|
|
|
trace_artifact&.destroy!
|
|
|
|
|
|
|
|
##
|
2018-07-04 01:37:33 -04:00
|
|
|
# Erase the live trace
|
2021-04-06 08:09:21 -04:00
|
|
|
erase_trace_chunks!
|
2018-07-03 01:00:25 -04:00
|
|
|
FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace
|
|
|
|
job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace
|
2018-07-04 01:51:12 -04:00
|
|
|
ensure
|
|
|
|
@current_path = nil
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
2018-02-23 07:08:38 -05:00
|
|
|
def archive!
|
2018-11-23 11:25:11 -05:00
|
|
|
in_write_lock do
|
2018-06-02 00:08:34 -04:00
|
|
|
unsafe_archive!
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-01-13 04:08:03 -05:00
|
|
|
def update_interval
|
2021-03-23 08:09:33 -04:00
|
|
|
if being_watched?
|
|
|
|
UPDATE_FREQUENCY_WHEN_BEING_WATCHED
|
|
|
|
else
|
2021-04-01 11:08:54 -04:00
|
|
|
UPDATE_FREQUENCY_DEFAULT
|
2021-03-23 08:09:33 -04:00
|
|
|
end
|
2020-01-13 04:08:03 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def being_watched!
|
|
|
|
Gitlab::Redis::SharedState.with do |redis|
|
|
|
|
redis.set(being_watched_cache_key, true, ex: WATCH_FLAG_TTL)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def being_watched?
|
|
|
|
Gitlab::Redis::SharedState.with do |redis|
|
|
|
|
redis.exists(being_watched_cache_key)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-09-28 11:09:44 -04:00
|
|
|
def lock(&block)
|
|
|
|
in_write_lock(&block)
|
|
|
|
rescue FailedToObtainLockError
|
|
|
|
raise LockedError, "build trace `#{job.id}` is locked"
|
|
|
|
end
|
|
|
|
|
2018-06-02 00:08:34 -04:00
|
|
|
private
|
|
|
|
|
2020-08-24 14:10:19 -04:00
|
|
|
def read_stream
|
|
|
|
stream = Gitlab::Ci::Trace::Stream.new do
|
|
|
|
if trace_artifact
|
|
|
|
trace_artifact.open
|
|
|
|
elsif job.trace_chunks.any?
|
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job)
|
|
|
|
elsif current_path
|
|
|
|
File.open(current_path, "rb")
|
|
|
|
elsif old_trace
|
|
|
|
StringIO.new(old_trace)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
yield stream
|
|
|
|
ensure
|
|
|
|
stream&.close
|
|
|
|
end
|
|
|
|
|
2018-11-23 11:25:11 -05:00
|
|
|
def unsafe_write!(mode, &blk)
|
|
|
|
stream = Gitlab::Ci::Trace::Stream.new do
|
|
|
|
if trace_artifact
|
|
|
|
raise AlreadyArchivedError, 'Could not write to the archived trace'
|
|
|
|
elsif current_path
|
|
|
|
File.open(current_path, mode)
|
2020-06-10 08:08:58 -04:00
|
|
|
elsif Feature.enabled?(:ci_enable_live_trace, job.project)
|
2018-11-23 11:25:11 -05:00
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job)
|
|
|
|
else
|
|
|
|
File.open(ensure_path, mode)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
yield(stream).tap do
|
|
|
|
job.touch if job.needs_touch?
|
|
|
|
end
|
|
|
|
ensure
|
|
|
|
stream&.close
|
|
|
|
end
|
|
|
|
|
2018-06-02 00:08:34 -04:00
|
|
|
def unsafe_archive!
|
2018-03-02 09:30:31 -05:00
|
|
|
raise ArchiveError, 'Job is not finished yet' unless job.complete?
|
2018-02-23 07:08:38 -05:00
|
|
|
|
2021-09-13 14:11:46 -04:00
|
|
|
unsafe_trace_conditionally_cleanup_before_retry!
|
2021-04-06 08:09:21 -04:00
|
|
|
|
2018-06-02 00:08:34 -04:00
|
|
|
if job.trace_chunks.any?
|
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
|
|
|
|
archive_stream!(stream)
|
2021-02-16 13:09:24 -05:00
|
|
|
destroy_stream(job) { stream.destroy! }
|
2018-06-02 00:08:34 -04:00
|
|
|
end
|
|
|
|
elsif current_path
|
|
|
|
File.open(current_path) do |stream|
|
|
|
|
archive_stream!(stream)
|
|
|
|
FileUtils.rm(current_path)
|
|
|
|
end
|
|
|
|
elsif old_trace
|
|
|
|
StringIO.new(old_trace, 'rb').tap do |stream|
|
|
|
|
archive_stream!(stream)
|
|
|
|
job.erase_old_trace!
|
2018-02-23 07:08:38 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2021-09-13 14:11:46 -04:00
|
|
|
def already_archived?
|
|
|
|
# TODO check checksum to ensure archive completed successfully
|
|
|
|
# See https://gitlab.com/gitlab-org/gitlab/-/issues/259619
|
|
|
|
trace_artifact.archived_trace_exists?
|
|
|
|
end
|
|
|
|
|
|
|
|
def unsafe_trace_conditionally_cleanup_before_retry!
|
2021-04-06 08:09:21 -04:00
|
|
|
return unless trace_artifact
|
|
|
|
|
2021-09-13 14:11:46 -04:00
|
|
|
if already_archived?
|
2021-04-06 08:09:21 -04:00
|
|
|
# An archive already exists, so make sure to remove the trace chunks
|
|
|
|
erase_trace_chunks!
|
2021-09-13 14:11:46 -04:00
|
|
|
raise AlreadyArchivedError, 'Could not archive again'
|
2021-04-06 08:09:21 -04:00
|
|
|
else
|
|
|
|
# An archive already exists, but its associated file does not, so remove it
|
|
|
|
trace_artifact.destroy!
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-11-23 11:25:11 -05:00
|
|
|
def in_write_lock(&blk)
|
2020-08-27 11:10:21 -04:00
|
|
|
lock_key = "trace:write:lock:#{job.id}"
|
2018-11-23 11:25:11 -05:00
|
|
|
in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk)
|
|
|
|
end
|
|
|
|
|
2018-02-23 07:08:38 -05:00
|
|
|
def archive_stream!(stream)
|
2021-10-04 11:12:14 -04:00
|
|
|
::Gitlab::Ci::Trace::Archive.new(job, trace_metadata).execute!(stream)
|
2021-09-06 14:11:17 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def trace_metadata
|
|
|
|
strong_memoize(:trace_metadata) do
|
|
|
|
job.ensure_trace_metadata!
|
2018-02-26 09:06:04 -05:00
|
|
|
end
|
2018-02-23 07:08:38 -05:00
|
|
|
end
|
|
|
|
|
2018-03-29 04:43:32 -04:00
|
|
|
def ensure_path
|
|
|
|
return current_path if current_path
|
|
|
|
|
|
|
|
ensure_directory
|
|
|
|
default_path
|
|
|
|
end
|
|
|
|
|
|
|
|
def ensure_directory
|
|
|
|
unless Dir.exist?(default_directory)
|
|
|
|
FileUtils.mkdir_p(default_directory)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-01-26 12:00:29 -05:00
|
|
|
def current_path
|
|
|
|
@current_path ||= paths.find do |trace_path|
|
|
|
|
File.exist?(trace_path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
def paths
|
2019-04-24 09:51:48 -04:00
|
|
|
[default_path]
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
2018-01-26 12:38:54 -05:00
|
|
|
def default_directory
|
2017-04-06 12:20:27 -04:00
|
|
|
File.join(
|
|
|
|
Settings.gitlab_ci.builds_path,
|
2018-01-26 12:38:54 -05:00
|
|
|
job.created_at.utc.strftime("%Y_%m"),
|
|
|
|
job.project_id.to_s
|
2017-04-06 12:20:27 -04:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2018-01-26 12:38:54 -05:00
|
|
|
def default_path
|
|
|
|
File.join(default_directory, "#{job.id}.log")
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
2018-01-25 04:50:56 -05:00
|
|
|
def trace_artifact
|
2021-02-16 13:09:24 -05:00
|
|
|
read_trace_artifact(job) { job.job_artifacts_trace }
|
|
|
|
end
|
|
|
|
|
2021-06-04 11:10:25 -04:00
|
|
|
def destroy_stream(build)
|
|
|
|
if consistent_archived_trace?(build)
|
|
|
|
::Gitlab::Database::LoadBalancing::Sticking
|
|
|
|
.stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id)
|
|
|
|
end
|
|
|
|
|
2021-02-16 13:09:24 -05:00
|
|
|
yield
|
|
|
|
end
|
|
|
|
|
2021-06-04 11:10:25 -04:00
|
|
|
def read_trace_artifact(build)
|
|
|
|
if consistent_archived_trace?(build)
|
|
|
|
::Gitlab::Database::LoadBalancing::Sticking
|
|
|
|
.unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id)
|
|
|
|
end
|
|
|
|
|
2021-02-16 13:09:24 -05:00
|
|
|
yield
|
2018-01-25 04:50:56 -05:00
|
|
|
end
|
2020-01-13 04:08:03 -05:00
|
|
|
|
2021-06-04 11:10:25 -04:00
|
|
|
def consistent_archived_trace?(build)
|
|
|
|
::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false)
|
|
|
|
end
|
|
|
|
|
2020-01-13 04:08:03 -05:00
|
|
|
def being_watched_cache_key
|
|
|
|
"gitlab:ci:trace:#{job.id}:watched"
|
|
|
|
end
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|