2017-04-06 12:20:27 -04:00
|
|
|
module Gitlab
|
|
|
|
module Ci
|
|
|
|
class Trace
|
2018-03-02 09:30:31 -05:00
|
|
|
ArchiveError = Class.new(StandardError)
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
attr_reader :job
|
|
|
|
|
|
|
|
delegate :old_trace, to: :job
|
|
|
|
|
|
|
|
def initialize(job)
|
|
|
|
@job = job
|
|
|
|
end
|
|
|
|
|
|
|
|
def html(last_lines: nil)
|
|
|
|
read do |stream|
|
|
|
|
stream.html(last_lines: last_lines)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def raw(last_lines: nil)
|
|
|
|
read do |stream|
|
|
|
|
stream.raw(last_lines: last_lines)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def extract_coverage(regex)
|
|
|
|
read do |stream|
|
|
|
|
stream.extract_coverage(regex)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-09-25 12:54:08 -04:00
|
|
|
def extract_sections
|
|
|
|
read do |stream|
|
|
|
|
stream.extract_sections
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
def set(data)
|
2018-04-05 03:16:01 -04:00
|
|
|
write('w+b') do |stream|
|
2017-04-06 12:20:27 -04:00
|
|
|
data = job.hide_secrets(data)
|
|
|
|
stream.set(data)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def append(data, offset)
|
2018-04-18 02:19:53 -04:00
|
|
|
write('a+b') do |stream|
|
2017-04-06 12:20:27 -04:00
|
|
|
current_length = stream.size
|
2018-05-06 13:54:41 -04:00
|
|
|
break current_length unless current_length == offset
|
2017-04-06 12:20:27 -04:00
|
|
|
|
|
|
|
data = job.hide_secrets(data)
|
|
|
|
stream.append(data, offset)
|
|
|
|
stream.size
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def exist?
|
2018-04-26 02:06:04 -04:00
|
|
|
trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def read
|
|
|
|
stream = Gitlab::Ci::Trace::Stream.new do
|
2018-01-26 12:38:54 -05:00
|
|
|
if trace_artifact
|
2018-01-25 04:50:56 -05:00
|
|
|
trace_artifact.open
|
2018-04-26 02:06:04 -04:00
|
|
|
elsif job.trace_chunks.any?
|
2018-04-04 06:19:17 -04:00
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job)
|
2018-01-25 04:50:56 -05:00
|
|
|
elsif current_path
|
2017-04-06 12:20:27 -04:00
|
|
|
File.open(current_path, "rb")
|
|
|
|
elsif old_trace
|
|
|
|
StringIO.new(old_trace)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
yield stream
|
|
|
|
ensure
|
|
|
|
stream&.close
|
|
|
|
end
|
|
|
|
|
2018-04-18 02:19:53 -04:00
|
|
|
def write(mode)
|
2017-04-06 12:20:27 -04:00
|
|
|
stream = Gitlab::Ci::Trace::Stream.new do
|
2018-04-04 06:19:17 -04:00
|
|
|
if current_path
|
2018-04-05 03:16:01 -04:00
|
|
|
File.open(current_path, mode)
|
2018-04-04 06:19:17 -04:00
|
|
|
elsif Feature.enabled?('ci_enable_live_trace')
|
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job)
|
2018-03-27 05:01:26 -04:00
|
|
|
else
|
2018-04-05 03:16:01 -04:00
|
|
|
File.open(ensure_path, mode)
|
2018-03-27 05:01:26 -04:00
|
|
|
end
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
yield(stream).tap do
|
|
|
|
job.touch if job.needs_touch?
|
|
|
|
end
|
|
|
|
ensure
|
|
|
|
stream&.close
|
|
|
|
end
|
|
|
|
|
|
|
|
def erase!
|
2018-01-26 12:38:54 -05:00
|
|
|
trace_artifact&.destroy
|
2018-01-25 08:46:38 -05:00
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
paths.each do |trace_path|
|
|
|
|
FileUtils.rm(trace_path, force: true)
|
|
|
|
end
|
|
|
|
|
2018-04-26 03:39:56 -04:00
|
|
|
job.trace_chunks.fast_destroy_all
|
2017-04-06 12:20:27 -04:00
|
|
|
job.erase_old_trace!
|
|
|
|
end
|
|
|
|
|
2018-02-23 07:08:38 -05:00
|
|
|
def archive!
|
2018-03-02 09:30:31 -05:00
|
|
|
raise ArchiveError, 'Already archived' if trace_artifact
|
|
|
|
raise ArchiveError, 'Job is not finished yet' unless job.complete?
|
2018-02-23 07:08:38 -05:00
|
|
|
|
2018-04-26 02:06:04 -04:00
|
|
|
if job.trace_chunks.any?
|
2018-04-04 06:19:17 -04:00
|
|
|
Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
|
|
|
|
archive_stream!(stream)
|
|
|
|
stream.destroy!
|
2018-03-28 13:13:08 -04:00
|
|
|
end
|
|
|
|
elsif current_path
|
2018-02-27 13:44:13 -05:00
|
|
|
File.open(current_path) do |stream|
|
|
|
|
archive_stream!(stream)
|
|
|
|
FileUtils.rm(current_path)
|
2018-02-23 07:08:38 -05:00
|
|
|
end
|
|
|
|
elsif old_trace
|
2018-02-27 13:44:13 -05:00
|
|
|
StringIO.new(old_trace, 'rb').tap do |stream|
|
2018-02-23 07:08:38 -05:00
|
|
|
archive_stream!(stream)
|
|
|
|
job.erase_old_trace!
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
private
|
|
|
|
|
2018-02-23 07:08:38 -05:00
|
|
|
def archive_stream!(stream)
|
2018-02-26 09:06:04 -05:00
|
|
|
clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
|
2018-04-26 02:06:04 -04:00
|
|
|
create_build_trace!(job, clone_path)
|
2018-02-26 09:06:04 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def clone_file!(src_stream, temp_dir)
|
|
|
|
FileUtils.mkdir_p(temp_dir)
|
|
|
|
Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path|
|
|
|
|
temp_path = File.join(dir_path, "job.log")
|
2018-02-27 13:44:13 -05:00
|
|
|
FileUtils.touch(temp_path)
|
|
|
|
size = IO.copy_stream(src_stream, temp_path)
|
2018-03-02 09:30:31 -05:00
|
|
|
raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size
|
2018-03-01 03:38:44 -05:00
|
|
|
|
2018-02-26 09:06:04 -05:00
|
|
|
yield(temp_path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-04-26 02:06:04 -04:00
|
|
|
def create_build_trace!(job, path)
|
2018-02-26 09:06:04 -05:00
|
|
|
File.open(path) do |stream|
|
|
|
|
job.create_job_artifacts_trace!(
|
|
|
|
project: job.project,
|
|
|
|
file_type: :trace,
|
2018-03-06 11:18:32 -05:00
|
|
|
file: stream,
|
|
|
|
file_sha256: Digest::SHA256.file(path).hexdigest)
|
2018-02-26 09:06:04 -05:00
|
|
|
end
|
2018-02-23 07:08:38 -05:00
|
|
|
end
|
|
|
|
|
2018-03-29 04:43:32 -04:00
|
|
|
def ensure_path
|
|
|
|
return current_path if current_path
|
|
|
|
|
|
|
|
ensure_directory
|
|
|
|
default_path
|
|
|
|
end
|
|
|
|
|
|
|
|
def ensure_directory
|
|
|
|
unless Dir.exist?(default_directory)
|
|
|
|
FileUtils.mkdir_p(default_directory)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-01-26 12:00:29 -05:00
|
|
|
def current_path
|
|
|
|
@current_path ||= paths.find do |trace_path|
|
|
|
|
File.exist?(trace_path)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-06 12:20:27 -04:00
|
|
|
def paths
|
|
|
|
[
|
2018-01-26 12:38:54 -05:00
|
|
|
default_path,
|
2017-04-06 12:20:27 -04:00
|
|
|
deprecated_path
|
|
|
|
].compact
|
|
|
|
end
|
|
|
|
|
2018-01-26 12:38:54 -05:00
|
|
|
def default_directory
|
2017-04-06 12:20:27 -04:00
|
|
|
File.join(
|
|
|
|
Settings.gitlab_ci.builds_path,
|
2018-01-26 12:38:54 -05:00
|
|
|
job.created_at.utc.strftime("%Y_%m"),
|
|
|
|
job.project_id.to_s
|
2017-04-06 12:20:27 -04:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2018-01-26 12:38:54 -05:00
|
|
|
def default_path
|
|
|
|
File.join(default_directory, "#{job.id}.log")
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def deprecated_path
|
|
|
|
File.join(
|
|
|
|
Settings.gitlab_ci.builds_path,
|
|
|
|
job.created_at.utc.strftime("%Y_%m"),
|
|
|
|
job.project.ci_id.to_s,
|
|
|
|
"#{job.id}.log"
|
|
|
|
) if job.project&.ci_id
|
|
|
|
end
|
2018-01-25 04:50:56 -05:00
|
|
|
|
|
|
|
def trace_artifact
|
|
|
|
job.job_artifacts_trace
|
|
|
|
end
|
2017-04-06 12:20:27 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|