From a1cddf051e6a4a2e0cb3a20efb9f323328fa4bce Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 6 May 2017 01:57:39 +0900 Subject: [PATCH 01/17] Reproduced --- spec/lib/gitlab/ci/trace/stream_spec.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index 40ac5a3ed37..a607f000431 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -240,10 +240,18 @@ describe Gitlab::Ci::Trace::Stream do end context 'multiple results in content & regex' do - let(:data) { ' (98.39%) covered. (98.29%) covered' } + let(:data) do + <<~HEREDOC + (98.39%) covered + (98.29%) covered + HEREDOC + end + let(:regex) { '\(\d+.\d+\%\) covered' } - it { is_expected.to eq("98.29") } + it 'returns the last matched coverage' do + is_expected.to eq("98.29") + end end context 'using a regex capture' do From 6018afa89fc34b0ea9ba691e3690a9837fadc122 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 6 May 2017 03:16:39 +0900 Subject: [PATCH 02/17] Add reverse_line --- lib/gitlab/ci/trace/stream.rb | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fa462cbe095..be8e40af016 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -73,7 +73,7 @@ module Gitlab match = "" - stream.each_line do |line| + reverse_line do |line| matches = line.scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -115,6 +115,26 @@ module Gitlab chunks.join.lines.last(last_lines).join end + + def reverse_line + pos = 0 + max = stream.size + + while true + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + yield(buf) + end + end end end end From ddf67424ec5d84ff373e5e39195b5012c84eb920 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 6 May 2017 17:38:56 +0900 Subject: [PATCH 03/17] Fix while true --- lib/gitlab/ci/trace/stream.rb | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index be8e40af016..72a0d669de0 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -117,23 +117,18 @@ module Gitlab end def reverse_line - pos = 0 + pos = BUFFER_SIZE max = stream.size - while true + while pos < max + stream.seek(-pos, IO::SEEK_END) + yield(stream.read(BUFFER_SIZE)) pos += BUFFER_SIZE - - buf = - if pos <= max - stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE) - else # Reached the head, read only left - stream.seek(0) - stream.read(BUFFER_SIZE - (pos - max)) - end - - yield(buf) end + + # Reached the head, read only left + stream.seek(0) + yield(stream.read(BUFFER_SIZE - (pos - max))) end end end From 3d11b7d8e25ca39b347e61e12100c7745b6ea085 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 10 May 2017 02:40:34 +0900 Subject: [PATCH 04/17] Fix reverse_line from chunk based --- lib/gitlab/ci/trace/stream.rb | 19 +++++++++++++++++-- spec/lib/gitlab/ci/trace/stream_spec.rb | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 72a0d669de0..546f6788a78 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -119,16 +119,31 @@ module Gitlab def reverse_line pos = BUFFER_SIZE max = stream.size + debris = '' while pos < max stream.seek(-pos, IO::SEEK_END) - yield(stream.read(BUFFER_SIZE)) + stream.read(BUFFER_SIZE).tap do |buf| + buf = buf + debris + lines = buf.split("\n") + lines.reverse.each do |line| + yield(line) + end + debris = lines.count > 0 ? lines[0] : '' + end pos += BUFFER_SIZE end # Reached the head, read only left stream.seek(0) - yield(stream.read(BUFFER_SIZE - (pos - max))) + last = (max > BUFFER_SIZE) ? (max % BUFFER_SIZE) : max + stream.read(last).tap do |buf| + buf = buf + debris + lines = buf.split("\n") + lines.reverse.each do |line| + yield(line) + end + end end end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index a607f000431..d90ccf4f799 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -254,6 +254,28 @@ describe Gitlab::Ci::Trace::Stream do end end + context 'when BUFFER_SIZE is smaller than stream.size' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + before do + stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5) + end + + it { is_expected.to eq("98.29") } + end + + context 'when BUFFER_SIZE is equal to stream.size' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + before do + stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', data.length) + end + + it { is_expected.to eq("98.29") } + end + context 'using a regex capture' do let(:data) { 'TOTAL 9926 3489 65%' } let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' } From 14c0833c1c9f1633ab6d3e791decd57eb7422c4f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 10 May 2017 02:44:47 +0900 Subject: [PATCH 05/17] Add changelog --- changelogs/unreleased/31556-ci-coverage-paralel-rspec.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/31556-ci-coverage-paralel-rspec.yml diff --git a/changelogs/unreleased/31556-ci-coverage-paralel-rspec.yml b/changelogs/unreleased/31556-ci-coverage-paralel-rspec.yml new file mode 100644 index 00000000000..4137050a077 --- /dev/null +++ b/changelogs/unreleased/31556-ci-coverage-paralel-rspec.yml @@ -0,0 +1,4 @@ +--- +title: Fix the last coverage in trace log should be extracted +merge_request: 11128 +author: dosuken123 From b9950c22b38456537af92c77210f1b837bdc914c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 16 May 2017 22:35:18 +0900 Subject: [PATCH 06/17] Use each_line. Avoid comparison of partial. Add UTF-8 spec. --- lib/gitlab/ci/trace/stream.rb | 8 +++----- spec/lib/gitlab/ci/trace/stream_spec.rb | 11 +++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 546f6788a78..56d6855c3c6 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -125,11 +125,10 @@ module Gitlab stream.seek(-pos, IO::SEEK_END) stream.read(BUFFER_SIZE).tap do |buf| buf = buf + debris - lines = buf.split("\n") - lines.reverse.each do |line| + debris, *lines = buf.each_line.to_a + lines.reverse_each do |line| yield(line) end - debris = lines.count > 0 ? lines[0] : '' end pos += BUFFER_SIZE end @@ -139,8 +138,7 @@ module Gitlab last = (max > BUFFER_SIZE) ? (max % BUFFER_SIZE) : max stream.read(last).tap do |buf| buf = buf + debris - lines = buf.split("\n") - lines.reverse.each do |line| + buf.each_line.reverse_each do |line| yield(line) end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index d90ccf4f799..b4fdfd06daf 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -265,6 +265,17 @@ describe Gitlab::Ci::Trace::Stream do it { is_expected.to eq("98.29") } end + context 'when regex is multi-byte char' do + let(:data) { 'ゴッドファット\n' } + let(:regex) { 'ゴッドファット' } + + before do + stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5) + end + + it { is_expected.to be_nil } + end + context 'when BUFFER_SIZE is equal to stream.size' do let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' } let(:regex) { '\(\d+.\d+\%\) covered' } From a6f6056c62693f18d9a2da570578b2e19aa20afe Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 15:05:18 +0900 Subject: [PATCH 07/17] Use force_encoding when regex contains UTF-8 char --- lib/gitlab/ci/trace/stream.rb | 2 +- spec/lib/gitlab/ci/trace/stream_spec.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 56d6855c3c6..233cc5c1c7a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -74,7 +74,7 @@ module Gitlab match = "" reverse_line do |line| - matches = line.scan(regex) + matches = line.force_encoding(Encoding.default_external).scan(regex) next unless matches.is_a?(Array) next if matches.empty? diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index b4fdfd06daf..bbb3f9912a3 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -266,14 +266,14 @@ describe Gitlab::Ci::Trace::Stream do end context 'when regex is multi-byte char' do - let(:data) { 'ゴッドファット\n' } - let(:regex) { 'ゴッドファット' } + let(:data) { '95.0 ゴッドファット\n' } + let(:regex) { '\d+\.\d+ ゴッドファット' } before do stub_const('Gitlab::Ci::Trace::Stream::BUFFER_SIZE', 5) end - it { is_expected.to be_nil } + it { is_expected.to eq('95.0') } end context 'when BUFFER_SIZE is equal to stream.size' do From 318b46407239f35cd11d35b21271fa743d435dad Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 17:53:31 +0900 Subject: [PATCH 08/17] Use force_encoding(regex.encoding) --- lib/gitlab/ci/trace/stream.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 233cc5c1c7a..3b4728951c4 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -74,7 +74,7 @@ module Gitlab match = "" reverse_line do |line| - matches = line.force_encoding(Encoding.default_external).scan(regex) + matches = line.force_encoding(regex.encoding).scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -86,7 +86,7 @@ module Gitlab nil rescue # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now + # so we just silently ignore error for now end private From dddfa28eec6a55f8548e848d41b4fa3749a7d4d7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 18:18:16 +0900 Subject: [PATCH 09/17] Refer reverse_line from read_last_lines --- lib/gitlab/ci/trace/stream.rb | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 3b4728951c4..3d32fae6525 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -91,29 +91,14 @@ module Gitlab private - def read_last_lines(last_lines) - chunks = [] - pos = lines = 0 - max = stream.size - - # We want an extra line to make sure fist line has full contents - while lines <= last_lines && pos < max - pos += BUFFER_SIZE - - buf = - if pos <= max - stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE) - else # Reached the head, read only left - stream.seek(0) - stream.read(BUFFER_SIZE - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) + def read_last_lines(limit) + result = '' + reverse_line do |line| + result = line + result + limit -= 1 + return result if limit <= 0 end - - chunks.join.lines.last(last_lines).join + result end def reverse_line From 6be609dd6418f1688d027f21cfe6fdb167375bc0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 18:26:38 +0900 Subject: [PATCH 10/17] much better --- lib/gitlab/ci/trace/stream.rb | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 3d32fae6525..5777332b251 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -92,13 +92,7 @@ module Gitlab private def read_last_lines(limit) - result = '' - reverse_line do |line| - result = line + result - limit -= 1 - return result if limit <= 0 - end - result + to_enum(:reverse_line).first(limit).reverse.join end def reverse_line From 8c1b07e0cc3e461939cae0a9566cbab172360cdf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 20:21:13 +0900 Subject: [PATCH 11/17] Optimize reverse_line --- lib/gitlab/ci/trace/stream.rb | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 5777332b251..e8c34f3da05 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -96,30 +96,31 @@ module Gitlab end def reverse_line - pos = BUFFER_SIZE + pos = 0 max = stream.size debris = '' - while pos < max + while (read_size = calc_read_size(pos, max)) > 0 + pos += read_size stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE).tap do |buf| + stream.read(read_size).tap do |buf| buf = buf + debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| yield(line) end end - pos += BUFFER_SIZE end - # Reached the head, read only left - stream.seek(0) - last = (max > BUFFER_SIZE) ? (max % BUFFER_SIZE) : max - stream.read(last).tap do |buf| - buf = buf + debris - buf.each_line.reverse_each do |line| - yield(line) - end + yield(debris) if debris != '' + end + + def calc_read_size(pos, max) + if pos > max + BUFFER_SIZE + (pos - max) + else + remain = max - pos + (remain > BUFFER_SIZE) ? BUFFER_SIZE : remain end end end From fdcd2f192dd37ac2a2ea5b73926122009fbbe0aa Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 20:39:47 +0900 Subject: [PATCH 12/17] Improve calc_read_size --- lib/gitlab/ci/trace/stream.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index e8c34f3da05..39826846be0 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -116,12 +116,8 @@ module Gitlab end def calc_read_size(pos, max) - if pos > max - BUFFER_SIZE + (pos - max) - else - remain = max - pos - (remain > BUFFER_SIZE) ? BUFFER_SIZE : remain - end + remain = max - pos + (remain > BUFFER_SIZE) ? BUFFER_SIZE : remain end end end From 72b0af2c4cefea51c521d3606a0a3307aee41348 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 May 2017 01:44:15 +0900 Subject: [PATCH 13/17] Avoid tap. Use unless&empty. --- lib/gitlab/ci/trace/stream.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 39826846be0..d64c43df1d5 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -103,16 +103,14 @@ module Gitlab while (read_size = calc_read_size(pos, max)) > 0 pos += read_size stream.seek(-pos, IO::SEEK_END) - stream.read(read_size).tap do |buf| - buf = buf + debris - debris, *lines = buf.each_line.to_a - lines.reverse_each do |line| - yield(line) - end + buf = stream.read(read_size) + debris + debris, *lines = buf.each_line.to_a + lines.reverse_each do |line| + yield(line) end end - yield(debris) if debris != '' + yield(debris) unless debris.empty? end def calc_read_size(pos, max) From 22f1d2a1c46ce9eb2835611c4eac6948bd983488 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 May 2017 01:59:04 +0900 Subject: [PATCH 14/17] Append gurad clause. remove unless debris.empty --- lib/gitlab/ci/trace/stream.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index d64c43df1d5..c2abe37f003 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -96,6 +96,8 @@ module Gitlab end def reverse_line + return if stream.size <= 0 + pos = 0 max = stream.size debris = '' @@ -110,7 +112,7 @@ module Gitlab end end - yield(debris) unless debris.empty? + yield(debris) end def calc_read_size(pos, max) From a16387486ae6db94b1aa49310cfc67642b68f23c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 22 May 2017 15:17:08 +0900 Subject: [PATCH 15/17] put force_encoding in def reverse_line --- lib/gitlab/ci/trace/stream.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c2abe37f003..45b32e4a7f2 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -74,7 +74,7 @@ module Gitlab match = "" reverse_line do |line| - matches = line.force_encoding(regex.encoding).scan(regex) + matches = line.scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -108,11 +108,11 @@ module Gitlab buf = stream.read(read_size) + debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| - yield(line) + yield(line.force_encoding('UTF-8')) end end - yield(debris) + yield(debris.force_encoding('UTF-8')) end def calc_read_size(pos, max) From cf4ab10d71ea03c83fad22abf70dac21bbf73e72 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 24 May 2017 20:20:20 +0900 Subject: [PATCH 16/17] Adopt ayufan script --- lib/gitlab/ci/trace/stream.rb | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 45b32e4a7f2..c5a1ddfe691 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -96,28 +96,29 @@ module Gitlab end def reverse_line - return if stream.size <= 0 - - pos = 0 - max = stream.size + stream.seek(0, IO::SEEK_END) debris = '' - while (read_size = calc_read_size(pos, max)) > 0 - pos += read_size - stream.seek(-pos, IO::SEEK_END) - buf = stream.read(read_size) + debris + while !(buf = read_backward(BUFFER_SIZE)).empty? + buf += debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| yield(line.force_encoding('UTF-8')) end end - yield(debris.force_encoding('UTF-8')) + yield(debris.force_encoding('UTF-8')) if !(debris).empty? end - def calc_read_size(pos, max) - remain = max - pos - (remain > BUFFER_SIZE) ? BUFFER_SIZE : remain + def read_backward(length) + cur_offset = stream.tell + start = cur_offset - length + start = 0 if start < 0 + + stream.seek(start, IO::SEEK_SET) + stream.read(cur_offset - start).tap do + stream.seek(start, IO::SEEK_SET) + end end end end From f9c9daac780666dbd2ce4ad6270c66e1ee32545a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 25 May 2017 01:35:40 +0900 Subject: [PATCH 17/17] Resolve static analysys. Use until instead of negative while --- lib/gitlab/ci/trace/stream.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c5a1ddfe691..c4c0623df6c 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -99,7 +99,7 @@ module Gitlab stream.seek(0, IO::SEEK_END) debris = '' - while !(buf = read_backward(BUFFER_SIZE)).empty? + until (buf = read_backward(BUFFER_SIZE)).empty? buf += debris debris, *lines = buf.each_line.to_a lines.reverse_each do |line| @@ -107,7 +107,7 @@ module Gitlab end end - yield(debris.force_encoding('UTF-8')) if !(debris).empty? + yield(debris.force_encoding('UTF-8')) unless debris.empty? end def read_backward(length)