kotovalexarian-likes-github/mperham--sidekiq
1
0
Fork 0
mirror of https://github.com/mperham/sidekiq.git synced 2022-11-09 13:52:34 -05:00
Code Releases Activity

Histogram chart for job-specific metrics (#5473)

Browse source 
* Refactor job query

* First attempt at a histogram chart

* Explore a box plot chart

* Show 3 chart variations

* Outline boxes instead of solid boxes

* Remove box plot chart

* Use linear y-axis

This matches the axis for the metrics overview page, and it clarifies the data.

* Data tables for job metrics

* Add histogram totals chart

* Move things around

* Tooltip for histogram chart

* Fix deploy tooltip

* Extract marks query

* Extract chart base class

* Renaming

* Ensure a min radius for histogram bubbles

High job counts can result in a very small multiplier, which was making some of the bubbles too small to be visible.

* Round everything to two decimals for consistency

* styling for metrics headers

* Show emdash when timing info is n/a

* No job results found message

* No need for metrics header
This commit is contained in:
Adam McCrea 2022-08-16 13:56:36 -04:00 • committed by GitHub
parent ff7e41924d
commit 0b3751bf29
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 343 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
lib/sidekiq/metrics/query.rb
View file

@ -60,57 +60,40 @@ module Sidekiq
time -= 60
end
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
result_range = result.starts_at..result.ends_at
marks.each do |timestamp, label|
time = Time.parse(timestamp)
if result_range.cover? time
result.marks << MarkResult.new(time, label)
end
end
result.marks = fetch_marks(result.starts_at..result.ends_at)
result
end
def for_job(klass)
resultset = {}
resultset[:date] = @time.to_date
resultset[:period] = :hour
resultset[:ends_at] = @time
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
def for_job(klass, minutes: 60)
result = Result.new
time = @time
initial = @pool.with do |conn|
redis_results = @pool.with do |conn|
conn.pipelined do |pipe|
resultset[:size] = 60
60.times do |idx|
key = "j|#{time.strftime("%Y%m%d|%-H:%-M")}"
minutes.times do |idx|
key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
result.prepend_bucket time
time -= 60
end
end
end
time = @time
hist = Histogram.new(klass)
results = @pool.with do |conn|
initial.map do |(ms, p, f)|
tm = Time.utc(time.year, time.month, time.mday, time.hour, time.min, 0)
{
time: tm.iso8601,
epoch: tm.to_i,
ms: ms.to_i, p: p.to_i, f: f.to_i, hist: hist.fetch(conn, time)
}.tap { |x|
x[:mark] = marks[x[:time]] if marks[x[:time]]
time -= 60
}
@pool.with do |conn|
redis_results.each do |(ms, p, f)|
result.job_results[klass].add_metric "ms", time, ms.to_i if ms
result.job_results[klass].add_metric "p", time, p.to_i if p
result.job_results[klass].add_metric "f", time, f.to_i if f
result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time)
time -= 60
end
end
resultset[:marks] = marks
resultset[:starts_at] = time
resultset[:data] = results
resultset
result.marks = fetch_marks(result.starts_at..result.ends_at)
result
end
class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
@ -128,20 +111,37 @@ module Sidekiq
end
end
class JobResult < Struct.new(:series, :totals)
class JobResult < Struct.new(:series, :hist, :totals)
def initialize
super
self.series = Hash.new { |h, k| h[k] = {} }
self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
self.hist = Hash.new { |h, k| h[k] = [] }
self.totals = Hash.new(0)
end
def add_metric(metric, time, value)
totals[metric] += value
series[metric][time.strftime("%H:%M")] = value
series[metric][time.strftime("%H:%M")] += value
# Include timing measurements in seconds for convenience
add_metric("s", time, value / 1000.0) if metric == "ms"
end
def add_hist(time, hist_result)
hist[time.strftime("%H:%M")] = hist_result
end
def total_avg(metric = "ms")
completed = totals["p"] - totals["f"]
totals[metric].to_f / completed
end
def series_avg(metric = "ms")
series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
completed = series.dig("p", bucket) - series.dig("f", bucket)
result[bucket] = completed == 0 ? 0 : value.to_f / completed
end
end
end
class MarkResult < Struct.new(:time, :label)
@ -149,6 +149,21 @@ module Sidekiq
time.strftime("%H:%M")
end
end
private
def fetch_marks(time_range)
[].tap do |result|
marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
marks.each do |timestamp, label|
time = Time.parse(timestamp)
if time_range.cover? time
result << MarkResult.new(time, label)
end
end
end
end
end
end
end

2
lib/sidekiq/web/application.rb
View file

@ -69,7 +69,7 @@ module Sidekiq
get "/metrics/:name" do
@name = route_params[:name]
q = Sidekiq::Metrics::Query.new
@resultset = q.for_job(@name)
@query_result = q.for_job(@name)
erb(:metrics_for_job)
end

58
test/test_metrics.rb
View file

@ -48,9 +48,9 @@ describe Sidekiq::Metrics do
q = Sidekiq::Metrics::Query.new(now: whence)
rs = q.for_job("FooJob")
refute_nil rs[:marks]
assert_equal 1, rs[:marks].size
assert_equal "cafed00d - some git summary line", rs[:marks][floor], rs.inspect
refute_nil rs.marks
assert_equal 1, rs.marks.size
assert_equal "cafed00d - some git summary line", rs.marks.first.label, rs.marks.inspect
d = Sidekiq::Metrics::Deploy.new
rs = d.fetch(whence)
@ -93,12 +93,12 @@ describe Sidekiq::Metrics do
q = Sidekiq::Metrics::Query.new(now: fixed_time)
result = q.top_jobs
assert_equal 60, result.buckets.size
assert_equal({}, result.job_results)
assert_equal([], result.job_results.keys)
q = Sidekiq::Metrics::Query.new(now: fixed_time)
rs = q.for_job("DoesntExist")
refute_nil rs
assert_equal 7, rs.size
result = q.for_job("DoesntExist")
assert_equal 60, result.buckets.size
assert_equal(["DoesntExist"], result.job_results.keys)
end
it "fetches top job data" do
@ -119,12 +119,15 @@ describe Sidekiq::Metrics do
assert_equal "22:03", result.buckets.last
assert_equal %w[App::SomeJob App::FooJob].sort, result.job_results.keys.sort
some_job_result = result.job_results["App::SomeJob"]
refute_nil some_job_result
assert_equal %w[p f ms s].sort, some_job_result.series.keys.sort
assert_equal %w[p f ms s].sort, some_job_result.totals.keys.sort
assert_equal 2, some_job_result.series.dig("p", "22:03")
assert_equal 3, some_job_result.totals["p"]
job_result = result.job_results["App::SomeJob"]
refute_nil job_result
assert_equal %w[p f ms s].sort, job_result.series.keys.sort
assert_equal %w[p f ms s].sort, job_result.totals.keys.sort
assert_equal 2, job_result.series.dig("p", "22:03")
assert_equal 3, job_result.totals["p"]
# Execution time is not consistent, so these assertions are not exact
assert job_result.total_avg("ms").between?(0.5, 2), job_result.total_avg("ms")
assert job_result.series_avg("s")["22:03"].between?(0.0005, 0.002), job_result.series_avg("s")
end
it "fetches job-specific data" do
@ -133,20 +136,27 @@ describe Sidekiq::Metrics do
d.mark(at: fixed_time - 300, label: "cafed00d - some git summary line")
q = Sidekiq::Metrics::Query.new(now: fixed_time)
rs = q.for_job("App::FooJob")
assert_equal Date.new(2022, 7, 22), rs[:date]
assert_equal 60, rs[:data].size
assert_equal ["2022-07-22T21:58:00Z", "cafed00d - some git summary line"], rs[:marks].first
result = q.for_job("App::FooJob")
assert_equal fixed_time - 59 * 60, result.starts_at
assert_equal fixed_time, result.ends_at
assert_equal 1, result.marks.size
assert_equal "cafed00d - some git summary line", result.marks[0].label
assert_equal "21:58", result.marks[0].bucket
data = rs[:data]
assert_equal({time: "2022-07-22T22:03:00Z", p: 1, f: 0}, data[0].slice(:time, :p, :f))
assert_equal({time: "2022-07-22T22:02:00Z", p: 3, f: 0}, data[1].slice(:time, :p, :f))
assert_equal "cafed00d - some git summary line", data[5][:mark]
assert_equal 60, result.buckets.size
assert_equal "21:04", result.buckets.first
assert_equal "22:03", result.buckets.last
# from create_known_data
hist = data[1][:hist]
assert_equal 2, hist[0]
assert_equal 1, hist[1]
assert_equal %w[App::FooJob], result.job_results.keys
job_result = result.job_results["App::FooJob"]
refute_nil job_result
assert_equal %w[p ms s].sort, job_result.series.keys.sort
assert_equal %w[p ms s].sort, job_result.totals.keys.sort
assert_equal 1, job_result.series.dig("p", "22:03")
assert_equal 4, job_result.totals["p"]
assert_equal 2, job_result.hist.dig("22:02", 0)
assert_equal 1, job_result.hist.dig("22:02", 1)
end
end
end

182
web/assets/javascripts/metrics.js
View file

@ -3,20 +3,17 @@ if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
Chart.defaults.color = "#aaa"
}
class MetricsChart {
class BaseChart {
constructor(id, options) {
this.ctx = document.getElementById(id);
this.series = options.series;
this.marks = options.marks;
this.labels = options.labels;
this.swatches = [];
this.options = options
this.fallbackColor = "#999";
this.colors = [
// Colors taken from https://www.chartjs.org/docs/latest/samples/utils.html
"#537bc4",
"#4dc9f6",
"#f67019",
"#f53794",
"#537bc4",
"#acc236",
"#166a8f",
"#00a950",
@ -25,15 +22,30 @@ class MetricsChart {
"#991b1b",
];
const datasets = Object.entries(this.series)
.filter(([kls, _]) => options.visible.includes(kls))
.map(([kls, _]) => this.dataset(kls));
this.chart = new Chart(this.ctx, {
type: "line",
data: { labels: this.labels, datasets: datasets },
type: this.options.chartType,
data: { labels: this.options.labels, datasets: this.datasets },
options: this.chartOptions,
});
}
addMarksToChart() {
this.options.marks.forEach(([bucket, label], i) => {
this.chart.options.plugins.annotation.annotations[`deploy-${i}`] = {
type: "line",
xMin: bucket,
xMax: bucket,
borderColor: "rgba(220, 38, 38, 0.4)",
borderWidth: 2,
};
});
}
}
class JobMetricsOverviewChart extends BaseChart {
constructor(id, options) {
super(id, { ...options, chartType: "line" });
this.swatches = [];
this.addMarksToChart();
this.chart.update();
@ -71,7 +83,7 @@ class MetricsChart {
return {
label: kls,
data: this.series[kls],
data: this.options.series[kls],
borderColor: color,
backgroundColor: color,
borderWidth: 2,
@ -79,16 +91,10 @@ class MetricsChart {
};
}
addMarksToChart() {
this.marks.forEach(([bucket, label], i) => {
this.chart.options.plugins.annotation.annotations[`deploy-${i}`] = {
type: "line",
xMin: bucket,
xMax: bucket,
borderColor: "rgba(220, 38, 38, 0.4)",
borderWidth: 2,
};
});
get datasets() {
return Object.entries(this.options.series)
.filter(([kls, _]) => this.options.visible.includes(kls))
.map(([kls, _]) => this.dataset(kls));
}
get chartOptions() {
@ -117,7 +123,135 @@ class MetricsChart {
`${item.dataset.label}: ${item.parsed.y.toFixed(1)} seconds`,
footer: (items) => {
const bucket = items[0].label;
const marks = this.marks.filter(([b, _]) => b == bucket);
const marks = this.options.marks.filter(([b, _]) => b == bucket);
return marks.map(([b, msg]) => `Deploy: ${msg}`);
},
},
},
},
};
}
}
class HistTotalsChart extends BaseChart {
constructor(id, options) {
super(id, { ...options, chartType: "bar" });
}
get datasets() {
return [{
data: this.options.series,
backgroundColor: this.colors[0],
borderWidth: 0,
}];
}
get chartOptions() {
return {
aspectRatio: 6,
scales: {
y: {
beginAtZero: true,
title: {
text: "Total jobs",
display: true,
},
},
},
interaction: {
mode: "x",
},
plugins: {
legend: {
display: false,
},
tooltip: {
callbacks: {
label: (item) => `${item.parsed.y} jobs`,
},
},
},
};
}
}
class HistBubbleChart extends BaseChart {
constructor(id, options) {
super(id, { ...options, chartType: "bubble" });
this.addMarksToChart();
this.chart.update();
}
get datasets() {
const data = [];
let maxCount = 0;
Object.entries(this.options.hist).forEach(([bucket, hist]) => {
hist.forEach((count, histBucket) => {
if (count > 0) {
data.push({
x: bucket,
// histogram data is ordered fastest to slowest, but this.histIntervals is
// slowest to fastest (so it displays correctly on the chart).
y:
this.options.histIntervals[this.options.histIntervals.length - 1 - histBucket] /
1000,
count: count,
});
if (count > maxCount) maxCount = count;
}
});
});
// Chart.js will not calculate the bubble size. We have to do that.
const maxRadius = this.ctx.offsetWidth / this.options.labels.length;
const minRadius = 1
const multiplier = (maxRadius / maxCount) * 1.5;
data.forEach((entry) => {
entry.r = entry.count * multiplier + minRadius;
});
return [{
data: data,
backgroundColor: "#537bc4",
borderColor: "#537bc4",
}];
}
get chartOptions() {
return {
aspectRatio: 3,
scales: {
x: {
type: "category",
labels: this.options.labels,
},
y: {
title: {
text: "Execution time (sec)",
display: true,
},
},
},
interaction: {
mode: "x",
},
plugins: {
legend: {
display: false,
},
tooltip: {
callbacks: {
title: (items) => `${items[0].raw.x} UTC`,
label: (item) =>
`${item.parsed.y} seconds: ${item.raw.count} job${
item.raw.count == 1 ? "" : "s"
}`,
footer: (items) => {
const bucket = items[0].raw.x;
const marks = this.options.marks.filter(([b, _]) => b == bucket);
return marks.map(([b, msg]) => `Deploy: ${msg}`);
},
},

11
web/assets/stylesheets/application.css
View file

@ -67,10 +67,15 @@ body {
padding: 0 20px;
}
h3 {
h1, h2, h3 {
font-size: 24px;
line-height: 45px;
}
.header-with-subheader h2 {
margin-top: -18px;
}
.centered {
text-align: center;
}
@ -988,3 +993,7 @@ div.interval-slider input {
outline: 1px solid #888;
outline-offset: 2px;
}
canvas {
margin: 20px 0 30px;
}

2
web/locales/en.yml
View file

@ -89,3 +89,5 @@ en: # <---- change this to your locale code
ExecutionTime: Total Execution Time
AvgExecutionTime: Average Execution Time
Context: Context
Bucket: Bucket
NoJobMetricsFound: No recent job metrics were found

18
web/views/metrics.erb
View file

@ -2,7 +2,7 @@
<script type="text/javascript" src="<%= root_path %>javascripts/chartjs-plugin-annotation.min.js"></script>
<script type="text/javascript" src="<%= root_path %>javascripts/metrics.js"></script>
<h3><%= t('Metrics') %></h3>
<h2>Total execution time</h2>
<%
table_limit = 20
@ -11,11 +11,11 @@
visible_kls = job_results.first(chart_limit).map(&:first)
%>
<canvas id="metrics-chart"></canvas>
<canvas id="job-metrics-overview-chart"></canvas>
<script>
window.metricsChart = new MetricsChart(
"metrics-chart",
window.jobMetricsChart = new JobMetricsOverviewChart(
"job-metrics-overview-chart",
<%= Sidekiq.dump_json({
series: job_results.map { |(kls, jr)| [kls, jr.dig("series", "s")] }.to_h,
marks: @query_result.marks.map { |m| [m.bucket, m.label] },
@ -25,7 +25,7 @@
)
</script>
<h3>Most Time-Consuming Jobs</h3>
<h2>Most Time-Consuming Jobs</h2>
<div class="table_container">
<table class="table table-bordered table-striped table-hover">
@ -51,12 +51,12 @@
/>
<code><a href="<%= root_path %>metrics/<%= kls %>"><%= kls %></a></code>
</div>
<script>metricsChart.registerSwatch("<%= id %>")</script>
<script>jobMetricsChart.registerSwatch("<%= id %>")</script>
</td>
<td><%= jr.dig("totals", "p") %></td>
<td><%= jr.dig("totals", "f") %></td>
<td><%= jr.dig("totals", "s").round(0) %> seconds</td>
<td><%= (jr.dig("totals", "s") / jr.dig("totals", "p")).round(2) %> seconds</td>
<td><%= jr.dig("totals", "s").round(2) %> seconds</td>
<td><%= jr.total_avg("s").round(2) %> seconds</td>
</tr>
<% end %>
<% else %>
@ -66,4 +66,4 @@
</table>
</div>
<p><small>Data from <%= @query_result.starts_at %> to <%= @query_result.ends_at %></small></p>
<p><small>Data from <%= @query_result.starts_at %> to <%= @query_result.ends_at %></small></p>

158
web/views/metrics_for_job.erb
View file

@ -1,92 +1,88 @@
<script type="text/javascript" src="<%= root_path %>javascripts/chart.min.js"></script>
<script type="text/javascript" src="<%= root_path %>javascripts/chartjs-plugin-annotation.min.js"></script>
<script type="text/javascript" src="<%= root_path %>javascripts/metrics.js"></script>
<h2><%= t('Metrics') %> / <%= h @name %></h2>
<%
job_result = @query_result.job_results[@name]
hist_totals = job_result.hist.values.first.zip(*job_result.hist.values[1..-1]).map(&:sum)
bucket_labels =Sidekiq::Metrics::Histogram::LABELS
bucket_intervals =Sidekiq::Metrics::Histogram::BUCKET_INTERVALS.reverse
<div class="row chart">
<div id="realtime" data-processed-label="<%= t('Processed') %>" data-failed-label="<%= t('Failed') %>"></div>
</div>
# Replace INFINITY since it can't be represented as JSON
bucket_intervals[0] = bucket_intervals[1] * 2
%>
<% data = @resultset[:data] %>
<div class="table_container">
<table class="table table-bordered table-striped table-hover">
<tbody>
<tr>
<th><%= t('Time') %></th>
<th><%= t('Processed') %></th>
<th><%= t('ExecutionTime') %></th>
<th><%= t('Failed') %></th>
<th><%= t('Deploy') %></th>
<th><%= t('Histogram') %></th>
</tr>
<% data.each do |hash| %>
<tr><td><%= hash[:time] %></td><td><%= hash[:p] %></td><td><%= hash[:ms] %></td><td><%= hash[:f] %></td><td><%= hash[:mark] %></td><td><%= hash[:hist] %></td></tr>
<% end %>
</tbody>
</table>
</div>
<% if job_result.totals["s"] > 0 %>
<div class="header-with-subheader">
<h1>
<a href="<%= root_path %>/metrics"><%= t(:metrics).to_s.titleize %></a> /
<%= h @name %>
</h1>
<h2>Histogram summary</h2>
</div>
<p>
Data from <%= @resultset[:starts_at] %> to <%= @resultset[:ends_at] %>
</p>
<canvas id="hist-totals-chart"></canvas>
<% atad = data.reverse %>
<script type="text/javascript" src="<%= root_path %>javascripts/graph.js"></script>
<script>
var palette = new Rickshaw.Color.Palette();
var data = [ {
name: "Processed",
color: palette.color(),
data: [ <% atad.each do |hash| %>
{ x: <%= hash[:epoch] %>, y: <%= hash[:p] %> },
<% end %> ]
}, {
name: "Failed",
color: palette.color(),
data: [ <% atad.each do |hash| %>
{ x: <%= hash[:epoch] %>, y: <%= hash[:f] %> },
<% end %>
]
}, {
name: "Execution Time",
color: palette.color(),
data: [ <% atad.each do |hash| %>
{ x: <%= hash[:epoch] %>, y: <%= hash[:ms] %> },
<% end %>
]
} ];
<script>
window.histTotalsChart = new HistTotalsChart(
"hist-totals-chart",
<%= Sidekiq.dump_json({
series: hist_totals,
labels: bucket_labels,
}) %>
)
</script>
// TODO What to do with this? Minutely hover detail with a histogram bar chart?
var histogramData = [ <% atad.each do |hash| %>
{ x: <%= hash[:epoch] %>, hist: <%= hash[:hist] %> },
<% end %> ]
var histogramLabels = <%= Sidekiq::Metrics::Histogram::LABELS.inspect %>;
<h2>Performance over time</h2>
var timeInterval = 60000;
var graphElement = document.getElementById("realtime");
<canvas id="hist-bubble-chart"></canvas>
var graph = new Rickshaw.Graph({
element: graphElement,
width: responsiveWidth(),
renderer: 'line',
interpolation: 'linear',
series: data,
});
var x_axis = new Rickshaw.Graph.Axis.Time( { graph: graph } );
<script>
window.histBubbleChart = new HistBubbleChart(
"hist-bubble-chart",
<%= Sidekiq.dump_json({
hist: job_result.hist,
marks: @query_result.marks.map { |m| [m.bucket, m.label] },
labels: @query_result.buckets,
histIntervals: bucket_intervals,
}) %>
)
</script>
var y_axis = new Rickshaw.Graph.Axis.Y( {
graph: graph,
tickFormat: Rickshaw.Fixtures.Number.formatKMBT,
ticksTreatment: 'glow'
});
<div class="table_container">
<table class="table table-bordered table-striped table-hover">
<tbody>
<tr>
<th><%= t('Time') %></th>
<th><%= t('Processed') %></th>
<th><%= t('Failed') %></th>
<th><%= t('ExecutionTime') %></th>
<th><%= t('AvgExecutionTime') %></th>
</tr>
<% @query_result.buckets.reverse.each do |bucket| %>
<tr>
<td><%= bucket %></td>
<td><%= job_result.series.dig("p", bucket) %></td>
<td><%= job_result.series.dig("f", bucket) %></td>
<% if (total_sec = job_result.series.dig("s", bucket)) > 0 %>
<td><%= total_sec.round(2) %> seconds</td>
<td><%= job_result.series_avg("s")[bucket].round(2) %> seconds</td>
<% else %>
<td>&mdash;</td>
<td>&mdash;</td>
<% end %>
</tr>
<% end %>
</tbody>
</table>
</div>
<% else %>
<h1>
<a href="<%= root_path %>/metrics"><%= t(:metrics).to_s.titleize %></a> /
<%= h @name %>
</h1>
graph.render();
<div class="alert alert-success"><%= t('NoJobMetricsFound') %></div>
<% end %>
var hoverDetail = new Rickshaw.Graph.HoverDetail( {
graph: graph,
// formatter: function(series, x, y) {
// var date = '<span class="date">' + new Date(x * 1000).toUTCString() + '</span>';
// var swatch = '<span class="detail_swatch" style="background-color: ' + series.color + '"></span>';
// var content = swatch + series.name + ": " + parseInt(y) + '<br>' + date;
// return content;
// }
} );
</script>
<p><small>Data from <%= @query_result.starts_at %> to <%= @query_result.ends_at %></small></p>