From d345591fc80e2181acfa71e9eeec99875c523767 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 16 Aug 2016 16:18:48 +0200 Subject: [PATCH] Tracking of custom events GitLab Performance Monitoring is now able to track custom events not directly related to application performance. These events include the number of tags pushed, repositories created, builds registered, etc. The use of these events is to get a better overview of how a GitLab instance is used and how that may affect performance. For example, a large number of Git pushes may have a negative impact on the underlying storage engine. Events are stored in the "events" measurement and are not prefixed with "rails_" or "sidekiq_", this makes it easier to query events with the same name triggered from different parts of the application. All events being stored in the same measurement also makes it easier to downsample data. Currently the following events are tracked: * Creating repositories * Removing repositories * Changing the default branch of a repository * Pushing a new tag * Removing an existing tag * Pushing a commit (along with the branch being pushed to) * Pushing a new branch * Removing an existing branch * Importing a repository (along with the URL we're importing) * Forking a repository (along with the source/target path) * CI builds registered (and when no build could be found) * CI builds being updated * Rails and Sidekiq exceptions Fixes gitlab-org/gitlab-ce#13720 --- CHANGELOG | 1 + app/models/repository.rb | 20 +++++++ app/workers/repository_fork_worker.rb | 4 ++ app/workers/repository_import_worker.rb | 4 ++ doc/monitoring/performance/influxdb_schema.md | 9 +++ lib/ci/api/builds.rb | 8 +++ lib/gitlab/metrics.rb | 9 +++ lib/gitlab/metrics/metric.rb | 9 ++- lib/gitlab/metrics/rack_middleware.rb | 4 ++ lib/gitlab/metrics/sidekiq_middleware.rb | 4 ++ lib/gitlab/metrics/transaction.rb | 21 ++++++- spec/lib/gitlab/metrics/metric_spec.rb | 18 ++++++ .../gitlab/metrics/rack_middleware_spec.rb | 9 +++ .../gitlab/metrics/sidekiq_middleware_spec.rb | 24 +++++++- spec/lib/gitlab/metrics/transaction_spec.rb | 57 +++++++++++++++++++ spec/lib/gitlab/metrics_spec.rb | 24 ++++++++ 16 files changed, 219 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 837e9e27aba..4986ffd7351 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,7 @@ v 8.11.0 (unreleased) - API: List access requests, request access, approve, and deny access requests to a project or a group. !4833 - Use long options for curl examples in documentation !5703 (winniehell) - Remove magic comments (`# encoding: UTF-8`) from Ruby files. !5456 (winniehell) + - GitLab Performance Monitoring can now track custom events such as the number of tags pushed to a repository - Add support for relative links starting with ./ or / to RelativeLinkFilter (winniehell) - Ignore URLs starting with // in Markdown links !5677 (winniehell) - Fix CI status icon link underline (ClemMakesApps) diff --git a/app/models/repository.rb b/app/models/repository.rb index e56bac509a4..916843dabdd 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -391,6 +391,8 @@ class Repository expire_exists_cache expire_root_ref_cache expire_emptiness_caches + + repository_event(:create_repository) end # Runs code just before a repository is deleted. @@ -407,6 +409,8 @@ class Repository expire_root_ref_cache expire_emptiness_caches expire_exists_cache + + repository_event(:remove_repository) end # Runs code just before the HEAD of a repository is changed. @@ -414,6 +418,8 @@ class Repository # Cached divergent commit counts are based on repository head expire_branch_cache expire_root_ref_cache + + repository_event(:change_default_branch) end # Runs code before pushing (= creating or removing) a tag. @@ -421,12 +427,16 @@ class Repository expire_cache expire_tags_cache expire_tag_count_cache + + repository_event(:push_tag) end # Runs code before removing a tag. def before_remove_tag expire_tags_cache expire_tag_count_cache + + repository_event(:remove_tag) end def before_import @@ -443,6 +453,8 @@ class Repository # Runs code after a new commit has been pushed. def after_push_commit(branch_name, revision) expire_cache(branch_name, revision) + + repository_event(:push_commit, branch: branch_name) end # Runs code after a new branch has been created. @@ -450,11 +462,15 @@ class Repository expire_branches_cache expire_has_visible_content_cache expire_branch_count_cache + + repository_event(:push_branch) end # Runs code before removing an existing branch. def before_remove_branch expire_branches_cache + + repository_event(:remove_branch) end # Runs code after an existing branch has been removed. @@ -1059,4 +1075,8 @@ class Repository def keep_around_ref_name(sha) "refs/keep-around/#{sha}" end + + def repository_event(event, tags = {}) + Gitlab::Metrics.add_event(event, { path: path_with_namespace }.merge(tags)) + end end diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d69d6037053..61ed1c38ac4 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -5,6 +5,10 @@ class RepositoryForkWorker sidekiq_options queue: :gitlab_shell def perform(project_id, forked_from_repository_storage_path, source_path, target_path) + Gitlab::Metrics.add_event(:fork_repository, + source_path: source_path, + target_path: target_path) + project = Project.find_by_id(project_id) unless project.present? diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 7d819fe78f8..e6701078f71 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -10,6 +10,10 @@ class RepositoryImportWorker @project = Project.find(project_id) @current_user = @project.creator + Gitlab::Metrics.add_event(:import_repository, + import_url: @project.import_url, + path: @project.path_with_namespace) + result = Projects::ImportService.new(project, current_user).execute if result[:status] == :error diff --git a/doc/monitoring/performance/influxdb_schema.md b/doc/monitoring/performance/influxdb_schema.md index 41861860b6d..eff0e29f58d 100644 --- a/doc/monitoring/performance/influxdb_schema.md +++ b/doc/monitoring/performance/influxdb_schema.md @@ -9,6 +9,7 @@ The following measurements are currently stored in InfluxDB: - `PROCESS_object_counts` - `PROCESS_transactions` - `PROCESS_views` +- `events` Here, `PROCESS` is replaced with either `rails` or `sidekiq` depending on the process type. In all series, any form of duration is stored in milliseconds. @@ -78,6 +79,14 @@ following value fields are available: The `action` tag contains the action name of the transaction that rendered the view. +## events + +This measurement is used to store generic events such as the number of Git +pushes, Emails sent, etc. Each point in this measurement has a single value +field called `count`. The value of this field is simply set to `1`. Each point +also has at least one tag: `event`. This tag's value is set to the event name. +Depending on the event type additional tags may be available as well. + --- Read more on: diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 260ac81f5fa..9f3b582a263 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -20,8 +20,13 @@ module Ci build = Ci::RegisterBuildService.new.execute(current_runner) if build + Gitlab::Metrics.add_event(:build_found, + project: build.project.path_with_namespace) + present build, with: Entities::BuildDetails else + Gitlab::Metrics.add_event(:build_not_found) + not_found! end end @@ -42,6 +47,9 @@ module Ci build.update_attributes(trace: params[:trace]) if params[:trace] + Gitlab::Metrics.add_event(:update_build, + project: build.project.path_with_namespace) + case params[:state].to_s when 'success' build.success diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 41fcd971c22..3d1ba33ec68 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -124,6 +124,15 @@ module Gitlab trans.action = action if trans end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + trans = current_transaction + + trans.add_event(*args) if trans + end + # Returns the prefix to use for the name of a series. def self.series_prefix @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f23d67e1e38..bd0afe53c51 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,15 +4,20 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags + attr_reader :series, :values, :tags, :type # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}) + def initialize(series, values, tags = {}, type = :metric) @values = values @series = series @tags = tags + @type = type + end + + def event? + type == :event end # Returns a Hash in a format that can be directly written to InfluxDB. diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e61670f491c..b4493bf44d2 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -17,6 +17,10 @@ module Gitlab begin retval = trans.run { @app.call(env) } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:rails_exception) + + raise error # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index a1240fd33ee..f9dd8e41912 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,10 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:sidekiq_exception) + + raise error ensure trans.finish end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 968f3218950..7bc16181be6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,10 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values, :methods + # The series to store events (e.g. Git pushes) in. + EVENT_SERIES = 'events' + + attr_reader :tags, :values, :method, :metrics attr_accessor :action @@ -55,6 +58,20 @@ module Gitlab @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) end + # Tracks a business level event + # + # Business level events including events such as Git pushes, Emails being + # sent, etc. + # + # event_name - The name of the event (e.g. "git_push"). + # tags - A set of tags to attach to the event. + def add_event(event_name, tags = {}) + @metrics << Metric.new(EVENT_SERIES, + { count: 1 }, + { event: event_name }.merge(tags), + :event) + end + # Returns a MethodCall object for the given name. def method_call_for(name) unless method = @methods[name] @@ -101,7 +118,7 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - hash[:tags][:action] ||= @action if @action + hash[:tags][:action] ||= @action if @action && !metric.event? hash end diff --git a/spec/lib/gitlab/metrics/metric_spec.rb b/spec/lib/gitlab/metrics/metric_spec.rb index f718d536130..f26fca52c50 100644 --- a/spec/lib/gitlab/metrics/metric_spec.rb +++ b/spec/lib/gitlab/metrics/metric_spec.rb @@ -23,6 +23,24 @@ describe Gitlab::Metrics::Metric do it { is_expected.to eq({ host: 'localtoast' }) } end + describe '#type' do + subject { metric.type } + + it { is_expected.to eq(:metric) } + end + + describe '#event?' do + it 'returns false for a regular metric' do + expect(metric.event?).to eq(false) + end + + it 'returns true for an event metric' do + expect(metric).to receive(:type).and_return(:event) + + expect(metric.event?).to eq(true) + end + end + describe '#to_hash' do it 'returns a Hash' do expect(metric.to_hash).to be_an_instance_of(Hash) diff --git a/spec/lib/gitlab/metrics/rack_middleware_spec.rb b/spec/lib/gitlab/metrics/rack_middleware_spec.rb index f264ed64029..a30cb2a5e38 100644 --- a/spec/lib/gitlab/metrics/rack_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/rack_middleware_spec.rb @@ -45,6 +45,15 @@ describe Gitlab::Metrics::RackMiddleware do middleware.call(env) end + + it 'tracks any raised exceptions' do + expect(app).to receive(:call).with(env).and_raise(RuntimeError) + + expect_any_instance_of(Gitlab::Metrics::Transaction). + to receive(:add_event).with(:rails_exception) + + expect { middleware.call(env) }.to raise_error(RuntimeError) + end end describe '#transaction_from_env' do diff --git a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb index 4d2aa03e722..acaba785606 100644 --- a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb @@ -12,7 +12,9 @@ describe Gitlab::Metrics::SidekiqMiddleware do with('TestWorker#perform'). and_call_original - expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float)) + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set). + with(:sidekiq_queue_duration, instance_of(Float)) + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish) middleware.call(worker, message, :test) { nil } @@ -25,10 +27,28 @@ describe Gitlab::Metrics::SidekiqMiddleware do with('TestWorker#perform'). and_call_original - expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float)) + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set). + with(:sidekiq_queue_duration, instance_of(Float)) + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish) middleware.call(worker, {}, :test) { nil } end + + it 'tracks any raised exceptions' do + worker = double(:worker, class: double(:class, name: 'TestWorker')) + + expect_any_instance_of(Gitlab::Metrics::Transaction). + to receive(:run).and_raise(RuntimeError) + + expect_any_instance_of(Gitlab::Metrics::Transaction). + to receive(:add_event).with(:sidekiq_exception) + + expect_any_instance_of(Gitlab::Metrics::Transaction). + to receive(:finish) + + expect { middleware.call(worker, message, :test) }. + to raise_error(RuntimeError) + end end end diff --git a/spec/lib/gitlab/metrics/transaction_spec.rb b/spec/lib/gitlab/metrics/transaction_spec.rb index f1a191d9410..3887c04c832 100644 --- a/spec/lib/gitlab/metrics/transaction_spec.rb +++ b/spec/lib/gitlab/metrics/transaction_spec.rb @@ -142,5 +142,62 @@ describe Gitlab::Metrics::Transaction do transaction.submit end + + it 'does not add an action tag for events' do + transaction.action = 'Foo#bar' + transaction.add_event(:meow) + + hash = { + series: 'events', + tags: { event: :meow }, + values: { count: 1 }, + timestamp: an_instance_of(Fixnum) + } + + expect(Gitlab::Metrics).to receive(:submit_metrics). + with([hash]) + + transaction.submit + end + end + + describe '#add_event' do + it 'adds a metric' do + transaction.add_event(:meow) + + expect(transaction.metrics[0]).to be_an_instance_of(Gitlab::Metrics::Metric) + end + + it "does not prefix the metric's series name" do + transaction.add_event(:meow) + + metric = transaction.metrics[0] + + expect(metric.series).to eq(described_class::EVENT_SERIES) + end + + it 'tracks a counter for every event' do + transaction.add_event(:meow) + + metric = transaction.metrics[0] + + expect(metric.values).to eq(count: 1) + end + + it 'tracks the event name' do + transaction.add_event(:meow) + + metric = transaction.metrics[0] + + expect(metric.tags).to eq(event: :meow) + end + + it 'allows tracking of custom tags' do + transaction.add_event(:meow, animal: 'cat') + + metric = transaction.metrics[0] + + expect(metric.tags).to eq(event: :meow, animal: 'cat') + end end end diff --git a/spec/lib/gitlab/metrics_spec.rb b/spec/lib/gitlab/metrics_spec.rb index 84f9475a0f8..ab6e311b1e8 100644 --- a/spec/lib/gitlab/metrics_spec.rb +++ b/spec/lib/gitlab/metrics_spec.rb @@ -153,4 +153,28 @@ describe Gitlab::Metrics do expect(described_class.series_prefix).to be_an_instance_of(String) end end + + describe '.add_event' do + context 'without a transaction' do + it 'does nothing' do + expect_any_instance_of(Gitlab::Metrics::Transaction). + not_to receive(:add_event) + + Gitlab::Metrics.add_event(:meow) + end + end + + context 'with a transaction' do + it 'adds an event' do + transaction = Gitlab::Metrics::Transaction.new + + expect(transaction).to receive(:add_event).with(:meow) + + expect(Gitlab::Metrics).to receive(:current_transaction). + and_return(transaction) + + Gitlab::Metrics.add_event(:meow) + end + end + end end