Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
GitLab Bot
parent
110225407f
commit
155f106fd5
|
|
@ -309,9 +309,7 @@ export default {
|
|||
this.isLatestVersion,
|
||||
);
|
||||
|
||||
if (this.glFeatures.usageDataDesignAction) {
|
||||
servicePingDesignDetailView();
|
||||
}
|
||||
servicePingDesignDetailView();
|
||||
},
|
||||
updateActiveDiscussion(id, source = ACTIVE_DISCUSSION_SOURCE_TYPES.discussion) {
|
||||
this.$apollo.mutate({
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ class Admin::UsersController < Admin::ApplicationController
|
|||
end
|
||||
|
||||
def user
|
||||
@user ||= find_routable!(User, params[:id], request.path_info)
|
||||
@user ||= find_routable!(User, params[:id], request.fullpath)
|
||||
end
|
||||
|
||||
def build_canonical_path(user)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
module ProjectUnauthorized
|
||||
module ControllerActions
|
||||
def self.on_routable_not_found
|
||||
lambda do |routable, path_info|
|
||||
lambda do |routable, full_path|
|
||||
return unless routable.is_a?(Project)
|
||||
|
||||
label = routable.external_authorization_classification_label
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@
|
|||
module RoutableActions
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
def find_routable!(routable_klass, routable_full_path, path_info, extra_authorization_proc: nil)
|
||||
def find_routable!(routable_klass, routable_full_path, full_path, extra_authorization_proc: nil)
|
||||
routable = routable_klass.find_by_full_path(routable_full_path, follow_redirects: request.get?)
|
||||
if routable_authorized?(routable, extra_authorization_proc)
|
||||
ensure_canonical_path(routable, routable_full_path)
|
||||
routable
|
||||
else
|
||||
perform_not_found_actions(routable, not_found_actions, path_info)
|
||||
perform_not_found_actions(routable, not_found_actions, full_path)
|
||||
|
||||
route_not_found unless performed?
|
||||
|
||||
|
|
@ -21,11 +21,11 @@ module RoutableActions
|
|||
[ProjectUnauthorized::ControllerActions.on_routable_not_found]
|
||||
end
|
||||
|
||||
def perform_not_found_actions(routable, actions, path_info)
|
||||
def perform_not_found_actions(routable, actions, full_path)
|
||||
actions.each do |action|
|
||||
break if performed?
|
||||
|
||||
instance_exec(routable, path_info, &action)
|
||||
instance_exec(routable, full_path, &action)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class Groups::ApplicationController < ApplicationController
|
|||
private
|
||||
|
||||
def group
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.path_info)
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.fullpath)
|
||||
end
|
||||
|
||||
def group_projects
|
||||
|
|
|
|||
|
|
@ -13,6 +13,6 @@ class Groups::Clusters::IntegrationsController < Clusters::IntegrationsControlle
|
|||
end
|
||||
|
||||
def group
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.path_info)
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.fullpath)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class Groups::ClustersController < Clusters::ClustersController
|
|||
end
|
||||
|
||||
def group
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.path_info)
|
||||
@group ||= find_routable!(Group, params[:group_id] || params[:id], request.fullpath)
|
||||
end
|
||||
|
||||
def metrics_dashboard_params
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ class Profiles::GroupsController < Profiles::ApplicationController
|
|||
feature_category :users
|
||||
|
||||
def update
|
||||
group = find_routable!(Group, params[:id], request.path_info)
|
||||
group = find_routable!(Group, params[:id], request.fullpath)
|
||||
notification_setting = current_user.notification_settings_for(group)
|
||||
|
||||
if notification_setting.update(update_params)
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class Projects::ApplicationController < ApplicationController
|
|||
path = File.join(params[:namespace_id], params[:project_id] || params[:id])
|
||||
auth_proc = ->(project) { !project.pending_delete? }
|
||||
|
||||
@project = find_routable!(Project, path, request.path_info, extra_authorization_proc: auth_proc)
|
||||
@project = find_routable!(Project, path, request.fullpath, extra_authorization_proc: auth_proc)
|
||||
end
|
||||
|
||||
def build_canonical_path(project)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,6 @@ class Projects::Clusters::IntegrationsController < ::Clusters::IntegrationsContr
|
|||
end
|
||||
|
||||
def project
|
||||
@project ||= find_routable!(Project, File.join(params[:namespace_id], params[:project_id]), request.path_info)
|
||||
@project ||= find_routable!(Project, File.join(params[:namespace_id], params[:project_id]), request.fullpath)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class Projects::ClustersController < Clusters::ClustersController
|
|||
end
|
||||
|
||||
def project
|
||||
@project ||= find_routable!(Project, File.join(params[:namespace_id], params[:project_id]), request.path_info)
|
||||
@project ||= find_routable!(Project, File.join(params[:namespace_id], params[:project_id]), request.fullpath)
|
||||
end
|
||||
|
||||
def repository
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ class Projects::IssuesController < Projects::ApplicationController
|
|||
before_action do
|
||||
push_frontend_feature_flag(:tribute_autocomplete, @project)
|
||||
push_frontend_feature_flag(:vue_issuables_list, project)
|
||||
push_frontend_feature_flag(:usage_data_design_action, project, default_enabled: true)
|
||||
push_frontend_feature_flag(:improved_emoji_picker, project, default_enabled: :yaml)
|
||||
push_frontend_feature_flag(:vue_issues_list, project)
|
||||
push_frontend_feature_flag(:iteration_cadences, project&.group, default_enabled: :yaml)
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class UsersController < ApplicationController
|
|||
private
|
||||
|
||||
def user
|
||||
@user ||= find_routable!(User, params[:username], request.path_info)
|
||||
@user ||= find_routable!(User, params[:username], request.fullpath)
|
||||
end
|
||||
|
||||
def personal_projects
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
---
|
||||
name: usage_data_design_action
|
||||
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46626
|
||||
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/287630
|
||||
milestone: '13.7'
|
||||
type: development
|
||||
group: group::product planning
|
||||
default_enabled: true
|
||||
|
|
@ -9,15 +9,19 @@ type: howto
|
|||
|
||||
Geo can be used in combination with Object Storage (AWS S3, or other compatible object storage).
|
||||
|
||||
The storage method for files is recorded in the database, and the database is replicated
|
||||
from the **primary** Geo site to the **secondary** Geo site, so the **secondary** Geo site
|
||||
must match the storage method of the **primary** Geo site.
|
||||
Therefore, if the **primary** Geo site uses object storage, the **secondary** Geo site must use it too.
|
||||
|
||||
Currently, **secondary** sites can use either:
|
||||
|
||||
- The same storage bucket as the **primary** site.
|
||||
- A replicated storage bucket.
|
||||
- Local storage, if the primary uses local storage.
|
||||
|
||||
The storage method (local or object storage) for files is recorded in the database, and the database
|
||||
is replicated from the **primary** Geo site to the **secondary** Geo site.
|
||||
|
||||
When accessing an uploaded object, we get its storage method (local or object storage) from the
|
||||
database, so the **secondary** Geo site must match the storage method of the **primary** Geo site.
|
||||
|
||||
Therefore, if the **primary** Geo site uses object storage, the **secondary** Geo site must use it too.
|
||||
|
||||
To have:
|
||||
|
||||
|
|
|
|||
|
|
@ -4,20 +4,680 @@ group: Product Intelligence
|
|||
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
|
||||
---
|
||||
|
||||
# Develop and test Service Ping
|
||||
# Implement Service Ping
|
||||
|
||||
To add a new metric and test Service Ping:
|
||||
Service Ping consists of two kinds of data:
|
||||
|
||||
- **Counters**: Track how often a certain event happened over time, such as how many CI/CD pipelines have run.
|
||||
They are monotonic and always trend up.
|
||||
- **Observations**: Facts collected from one or more GitLab instances and can carry arbitrary data.
|
||||
There are no general guidelines for how to collect those, due to the individual nature of that data.
|
||||
|
||||
To implement a new metric in Service Ping, follow these steps:
|
||||
|
||||
1. [Implement the required counter](#types-of-counters)
|
||||
1. [Name and place the metric](#name-and-place-the-metric)
|
||||
1. [Test counters manually using your Rails console](#test-counters-manually-using-your-rails-console)
|
||||
1. [Generate the SQL query](#generate-the-sql-query)
|
||||
1. [Optimize queries with `#database-lab`](#optimize-queries-with-database-lab)
|
||||
1. [Add the metric definition](#add-the-metric-definition)
|
||||
1. [Add the metric definition to the Metrics Dictionary](#add-the-metric-definition)
|
||||
1. [Add the metric to the Versions Application](#add-the-metric-to-the-versions-application)
|
||||
1. [Create a merge request](#create-a-merge-request)
|
||||
1. [Verify your metric](#verify-your-metric)
|
||||
1. [Set up and test Service Ping locally](#set-up-and-test-service-ping-locally)
|
||||
|
||||
## Instrumentation classes
|
||||
|
||||
We recommend you use [instrumentation classes](metrics_instrumentation.md) in `usage_data.rb` where possible.
|
||||
|
||||
For example, we have the following instrumentation class:
|
||||
`lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb`.
|
||||
|
||||
You should add it to `usage_data.rb` as follows:
|
||||
|
||||
```ruby
|
||||
boards: add_metric('CountBoardsMetric', time_frame: 'all'),
|
||||
```
|
||||
|
||||
## Types of counters
|
||||
|
||||
There are several types of counters in `usage_data.rb`:
|
||||
|
||||
- **[Batch counters](#batch-counters)**: Used for counts and sums.
|
||||
- **[Redis counters](#redis-counters):** Used for in-memory counts.
|
||||
- **[Alternative counters](#alternative-counters):** Used for settings and configurations.
|
||||
|
||||
NOTE:
|
||||
Only use the provided counter methods. Each counter method contains a built-in fail-safe mechanism that isolates each counter to avoid breaking the entire Service Ping process.
|
||||
|
||||
### Batch counters
|
||||
|
||||
For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query.
|
||||
|
||||
For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables:
|
||||
|
||||
| Table | Row counts in millions |
|
||||
|------------------------------|------------------------|
|
||||
| `merge_request_diff_commits` | 2280 |
|
||||
| `ci_build_trace_sections` | 1764 |
|
||||
| `merge_request_diff_files` | 1082 |
|
||||
| `events` | 514 |
|
||||
|
||||
Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases,
|
||||
you must add a specialized index on the columns involved in a counter.
|
||||
|
||||
#### Ordinary batch counters
|
||||
|
||||
Simple count of a given `ActiveRecord_Relation`, does a non-distinct batch count, smartly reduces `batch_size`, and handles errors.
|
||||
Handles the `ActiveRecord::StatementInvalid` error.
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
count(relation, column = nil, batch: true, start: nil, finish: nil)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation` the ActiveRecord_Relation to perform the count
|
||||
- `column` the column to perform the count on, by default is the primary key
|
||||
- `batch`: default `true` to use batch counting
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
count(User.active)
|
||||
count(::Clusters::Cluster.aws_installed.enabled, :cluster_id)
|
||||
count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id))
|
||||
```
|
||||
|
||||
#### Distinct batch counters
|
||||
|
||||
Distinct count of a given `ActiveRecord_Relation` on given column, a distinct batch count, smartly reduces `batch_size`, and handles errors.
|
||||
Handles the `ActiveRecord::StatementInvalid` error.
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation`: the ActiveRecord_Relation to perform the count
|
||||
- `column`: the column to perform the distinct count, by default is the primary key
|
||||
- `batch`: default `true` to use batch counting
|
||||
- `batch_size`: if none set it uses default value 10000 from `Gitlab::Database::BatchCounter`
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
WARNING:
|
||||
Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../iterating_tables_in_batches.md) guide.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
distinct_count(::Project, :creator_id)
|
||||
distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id))
|
||||
distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id')
|
||||
```
|
||||
|
||||
#### Sum batch operation
|
||||
|
||||
Sum the values of a given ActiveRecord_Relation on given column and handles errors.
|
||||
Handles the `ActiveRecord::StatementInvalid` error
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
sum(relation, column, batch_size: nil, start: nil, finish: nil)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation`: the ActiveRecord_Relation to perform the operation
|
||||
- `column`: the column to sum on
|
||||
- `batch_size`: if none set it uses default value 1000 from `Gitlab::Database::BatchCounter`
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
sum(JiraImportState.finished, :imported_issues_count)
|
||||
```
|
||||
|
||||
#### Grouping and batch operations
|
||||
|
||||
The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation`
|
||||
object, which groups by a specified column. With a grouped relation, the methods do batch counting,
|
||||
handle errors, and returns a hash table of key-value pairs.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
count(Namespace.group(:type))
|
||||
# returns => {nil=>179, "Group"=>54}
|
||||
|
||||
distinct_count(Project.group(:visibility_level), :creator_id)
|
||||
# returns => {0=>1, 10=>1, 20=>11}
|
||||
|
||||
sum(Issue.group(:state_id), :weight))
|
||||
# returns => {1=>3542, 2=>6820}
|
||||
```
|
||||
|
||||
#### Add operation
|
||||
|
||||
Sum the values given as parameters. Handles the `StandardError`.
|
||||
Returns `-1` if any of the arguments are `-1`.
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
add(*args)
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
project_imports = distinct_count(::Project.where.not(import_type: nil), :creator_id)
|
||||
bulk_imports = distinct_count(::BulkImport, :user_id)
|
||||
|
||||
add(project_imports, bulk_imports)
|
||||
```
|
||||
|
||||
#### Estimated batch counters
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48233) in GitLab 13.7.
|
||||
|
||||
Estimated batch counter functionality handles `ActiveRecord::StatementInvalid` errors
|
||||
when used through the provided `estimate_batch_distinct_count` method.
|
||||
Errors return a value of `-1`.
|
||||
|
||||
WARNING:
|
||||
This functionality estimates a distinct count of a specific ActiveRecord_Relation in a given column,
|
||||
which uses the [HyperLogLog](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) algorithm.
|
||||
As the HyperLogLog algorithm is probabilistic, the **results always include error**.
|
||||
The highest encountered error rate is 4.9%.
|
||||
|
||||
When correctly used, the `estimate_batch_distinct_count` method enables efficient counting over
|
||||
columns that contain non-unique values, which can not be assured by other counters.
|
||||
|
||||
##### estimate_batch_distinct_count method
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
|
||||
```
|
||||
|
||||
The [method](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/utils/usage_data.rb#L63)
|
||||
includes the following arguments:
|
||||
|
||||
- `relation`: The ActiveRecord_Relation to perform the count.
|
||||
- `column`: The column to perform the distinct count. The default is the primary key.
|
||||
- `batch_size`: From `Gitlab::Database::PostgresHll::BatchDistinctCounter::DEFAULT_BATCH_SIZE`. Default value: 10,000.
|
||||
- `start`: The custom start of the batch count, to avoid complex minimum calculations.
|
||||
- `finish`: The custom end of the batch count to avoid complex maximum calculations.
|
||||
|
||||
The method includes the following prerequisites:
|
||||
|
||||
- The supplied `relation` must include the primary key defined as the numeric column.
|
||||
For example: `id bigint NOT NULL`.
|
||||
- The `estimate_batch_distinct_count` can handle a joined relation. To use its ability to
|
||||
count non-unique columns, the joined relation **must not** have a one-to-many relationship,
|
||||
such as `has_many :boards`.
|
||||
- Both `start` and `finish` arguments should always represent primary key relationship values,
|
||||
even if the estimated count refers to another column, for example:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Note, :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id))
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
1. Simple execution of estimated batch counter, with only relation provided,
|
||||
returned value represents estimated number of unique values in `id` column
|
||||
(which is the primary key) of `Project` relation:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Project)
|
||||
```
|
||||
|
||||
1. Execution of estimated batch counter, where provided relation has applied
|
||||
additional filter (`.where(time_period)`), number of unique values estimated
|
||||
in custom column (`:author_id`), and parameters: `start` and `finish` together
|
||||
apply boundaries that defines range of provided relation to analyze:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id))
|
||||
```
|
||||
|
||||
1. Execution of estimated batch counter with joined relation (`joins(:cluster)`),
|
||||
for a custom column (`'clusters.user_id'`):
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id')
|
||||
```
|
||||
|
||||
When instrumenting metric with usage of estimated batch counter please add
|
||||
`_estimated` suffix to its name, for example:
|
||||
|
||||
```ruby
|
||||
"counts": {
|
||||
"ci_builds_estimated": estimate_batch_distinct_count(Ci::Build),
|
||||
...
|
||||
```
|
||||
|
||||
### Redis counters
|
||||
|
||||
Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent`.
|
||||
Returns -1 when a block is sent or hash with all values and -1 when a `counter(Gitlab::UsageDataCounters)` is sent.
|
||||
The different behavior is due to 2 different implementations of the Redis counter.
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
redis_usage_data(counter, &block)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented
|
||||
- or a `block`: which is evaluated
|
||||
|
||||
#### Ordinary Redis counters
|
||||
|
||||
Examples of implementation:
|
||||
|
||||
- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb)
|
||||
- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb)
|
||||
|
||||
##### UsageData API tracking
|
||||
|
||||
<!-- There's nearly identical content in `##### Adding new events`. If you fix errors here, you may need to fix the same errors in the other location. -->
|
||||
|
||||
1. Track event using `UsageData` API
|
||||
|
||||
Increment event count using ordinary Redis counter, for given event name.
|
||||
|
||||
Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default.
|
||||
|
||||
API requests are protected by checking for a valid CSRF token.
|
||||
|
||||
To be able to increment the values, the related feature `usage_data_<event_name>` should be enabled.
|
||||
|
||||
```plaintext
|
||||
POST /usage_data/increment_counter
|
||||
```
|
||||
|
||||
| Attribute | Type | Required | Description |
|
||||
| :-------- | :--- | :------- | :---------- |
|
||||
| `event` | string | yes | The event name it should be tracked |
|
||||
|
||||
Response:
|
||||
|
||||
- `200` if event was tracked
|
||||
- `400 Bad request` if event parameter is missing
|
||||
- `401 Unauthorized` if user is not authenticated
|
||||
- `403 Forbidden` for invalid CSRF token provided
|
||||
|
||||
1. Track events using JavaScript/Vue API helper which calls the API above
|
||||
|
||||
Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled to be able to track events
|
||||
|
||||
```javascript
|
||||
import api from '~/api';
|
||||
|
||||
api.trackRedisCounterEvent('my_already_defined_event_name'),
|
||||
```
|
||||
|
||||
#### Redis HLL counters
|
||||
|
||||
WARNING:
|
||||
HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes some small error**. According to [Redis documentation](https://redis.io/commands/pfcount), data from
|
||||
used HLL implementation is "approximated with a standard error of 0.81%".
|
||||
|
||||
With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values.
|
||||
|
||||
Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount).
|
||||
|
||||
##### Add new events
|
||||
|
||||
1. Define events in [`known_events`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/).
|
||||
|
||||
Example event:
|
||||
|
||||
```yaml
|
||||
- name: users_creating_epics
|
||||
category: epics_usage
|
||||
redis_slot: users
|
||||
aggregation: weekly
|
||||
feature_flag: track_epics_activity
|
||||
```
|
||||
|
||||
Keys:
|
||||
|
||||
- `name`: unique event name.
|
||||
|
||||
Name format for Redis HLL events `<name>_<redis_slot>`.
|
||||
|
||||
[See Metric name](metrics_dictionary.md#metric-name) for a complete guide on metric naming suggestion.
|
||||
|
||||
Consider including in the event's name the Redis slot to be able to count totals for a specific category.
|
||||
|
||||
Example names: `users_creating_epics`, `users_triggering_security_scans`.
|
||||
|
||||
- `category`: event category. Used for getting total counts for events in a category, for easier
|
||||
access to a group of events.
|
||||
- `redis_slot`: optional Redis slot. Default value: event name. Only event data that is stored in the same slot
|
||||
can be aggregated. Ensure keys are in the same slot. For example:
|
||||
`users_creating_epics` with `redis_slot: 'users'` builds Redis key
|
||||
`{users}_creating_epics-2020-34`. If `redis_slot` is not defined the Redis key will
|
||||
be `{users_creating_epics}-2020-34`.
|
||||
Recommended slots to use are: `users`, `projects`. This is the value we count.
|
||||
- `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly
|
||||
aggregation.
|
||||
- `aggregation`: may be set to a `:daily` or `:weekly` key. Defines how counting data is stored in Redis.
|
||||
Aggregation on a `daily` basis does not pull more fine grained data.
|
||||
- `feature_flag`: optional `default_enabled: :yaml`. If no feature flag is set then the tracking is enabled. One feature flag can be used for multiple events. For details, see our [GitLab internal Feature flags](../feature_flags/index.md) documentation. The feature flags are owned by the group adding the event tracking.
|
||||
|
||||
1. Use one of the following methods to track the event:
|
||||
|
||||
- In the controller using the `RedisTracking` module and the following format:
|
||||
|
||||
```ruby
|
||||
track_redis_hll_event(*controller_actions, name:, if: nil, &block)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `controller_actions`: the controller actions to track.
|
||||
- `name`: the event name.
|
||||
- `if`: optional custom conditions. Uses the same format as Rails callbacks.
|
||||
- `&block`: optional block that computes and returns the `custom_id` that we want to track. This overrides the `visitor_id`.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
# controller
|
||||
class ProjectsController < Projects::ApplicationController
|
||||
include RedisTracking
|
||||
|
||||
skip_before_action :authenticate_user!, only: :show
|
||||
track_redis_hll_event :index, :show, name: 'users_visiting_projects'
|
||||
|
||||
def index
|
||||
render html: 'index'
|
||||
end
|
||||
|
||||
def new
|
||||
render html: 'new'
|
||||
end
|
||||
|
||||
def show
|
||||
render html: 'show'
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
- In the API using the `increment_unique_values(event_name, values)` helper method.
|
||||
|
||||
Arguments:
|
||||
|
||||
- `event_name`: the event name.
|
||||
- `values`: the values counted. Can be one value or an array of values.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
get ':id/registry/repositories' do
|
||||
repositories = ContainerRepositoriesFinder.new(
|
||||
user: current_user, subject: user_group
|
||||
).execute
|
||||
|
||||
increment_unique_values('users_listing_repositories', current_user.id)
|
||||
|
||||
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
|
||||
end
|
||||
```
|
||||
|
||||
- Using `track_usage_event(event_name, values)` in services and GraphQL.
|
||||
|
||||
Increment unique values count using Redis HLL, for a given event name.
|
||||
|
||||
Examples:
|
||||
|
||||
- [Track usage event for an incident in a service](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/app/services/issues/update_service.rb#L66)
|
||||
- [Track usage event for an incident in GraphQL](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/app/graphql/mutations/alert_management/update_alert_status.rb#L16)
|
||||
|
||||
```ruby
|
||||
track_usage_event(:incident_management_incident_created, current_user.id)
|
||||
```
|
||||
|
||||
- Using the `UsageData` API.
|
||||
<!-- There's nearly identical content in `##### UsageData API Tracking`. If you find / fix errors here, you may need to fix errors in that section too. -->
|
||||
|
||||
Increment unique users count using Redis HLL, for a given event name.
|
||||
|
||||
To track events using the `UsageData` API, ensure the `usage_data_api` feature flag
|
||||
is set to `default_enabled: true`. Enabled by default in GitLab 13.7 and later.
|
||||
|
||||
API requests are protected by checking for a valid CSRF token.
|
||||
|
||||
```plaintext
|
||||
POST /usage_data/increment_unique_users
|
||||
```
|
||||
|
||||
| Attribute | Type | Required | Description |
|
||||
| :-------- | :--- | :------- | :---------- |
|
||||
| `event` | string | yes | The event name to track |
|
||||
|
||||
Response:
|
||||
|
||||
- `200` if the event was tracked, or if tracking failed for any reason.
|
||||
- `400 Bad request` if an event parameter is missing.
|
||||
- `401 Unauthorized` if the user is not authenticated.
|
||||
- `403 Forbidden` if an invalid CSRF token is provided.
|
||||
|
||||
- Using the JavaScript/Vue API helper, which calls the `UsageData` API.
|
||||
|
||||
To track events using the `UsageData` API, ensure the `usage_data_api` feature flag
|
||||
is set to `default_enabled: true`. Enabled by default in GitLab 13.7 and later.
|
||||
|
||||
Example for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/):
|
||||
|
||||
```javascript
|
||||
import api from '~/api';
|
||||
|
||||
api.trackRedisHllUserEvent('my_already_defined_event_name'),
|
||||
```
|
||||
|
||||
1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date:, context: '')`.
|
||||
|
||||
Arguments:
|
||||
|
||||
- `event_names`: the list of event names.
|
||||
- `start_date`: start date of the period for which we want to get event data.
|
||||
- `end_date`: end date of the period for which we want to get event data.
|
||||
- `context`: context of the event. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate`.
|
||||
|
||||
1. Testing tracking and getting unique events
|
||||
|
||||
Trigger events in rails console by using `track_event` method
|
||||
|
||||
```ruby
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_viewing_compliance_audit_events', values: 1)
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_viewing_compliance_audit_events', values: [2, 3])
|
||||
```
|
||||
|
||||
Next, get the unique events for the current week.
|
||||
|
||||
```ruby
|
||||
# Get unique events for metric for current_week
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_viewing_compliance_audit_events',
|
||||
start_date: Date.current.beginning_of_week, end_date: Date.current.next_week)
|
||||
```
|
||||
|
||||
##### Recommendations
|
||||
|
||||
We have the following recommendations for [adding new events](#add-new-events):
|
||||
|
||||
- Event aggregation: weekly.
|
||||
- Key expiry time:
|
||||
- Daily: 29 days.
|
||||
- Weekly: 42 days.
|
||||
- When adding new metrics, use a [feature flag](../../operations/feature_flags.md) to control the impact.
|
||||
- For feature flags triggered by another service, set `default_enabled: false`,
|
||||
- Events can be triggered using the `UsageData` API, which helps when there are > 10 events per change
|
||||
|
||||
##### Enable or disable Redis HLL tracking
|
||||
|
||||
Events are tracked behind optional [feature flags](../feature_flags/index.md) due to concerns for Redis performance and scalability.
|
||||
|
||||
For a full list of events and corresponding feature flags see, [known_events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files.
|
||||
|
||||
To enable or disable tracking for specific event in <https://gitlab.com> or <https://about.staging.gitlab.com>, run commands such as the following to
|
||||
[enable or disable the corresponding feature](../feature_flags/index.md).
|
||||
|
||||
```shell
|
||||
/chatops run feature set <feature_name> true
|
||||
/chatops run feature set <feature_name> false
|
||||
```
|
||||
|
||||
We can also disable tracking completely by using the global flag:
|
||||
|
||||
```shell
|
||||
/chatops run feature set redis_hll_tracking true
|
||||
/chatops run feature set redis_hll_tracking false
|
||||
```
|
||||
|
||||
##### Known events are added automatically in Service Data payload
|
||||
|
||||
All events added in [`known_events/common.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml) are automatically added to Service Data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209).
|
||||
For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable:
|
||||
|
||||
- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#add-new-events) events and data for the last complete week for weekly [aggregation](#add-new-events) events.
|
||||
- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#add-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#add-new-events) events.
|
||||
|
||||
Redis HLL implementation calculates automatic total metrics, if there are more than one metric for the same category, aggregation, and Redis slot.
|
||||
|
||||
- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric.
|
||||
- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric.
|
||||
|
||||
Example of `redis_hll_counters` data:
|
||||
|
||||
```ruby
|
||||
{:redis_hll_counters=>
|
||||
{"compliance"=>
|
||||
{"users_viewing_compliance_dashboard_weekly"=>0,
|
||||
"users_viewing_compliance_dashboard_monthly"=>0,
|
||||
"users_viewing_compliance_audit_events_weekly"=>0,
|
||||
"users_viewing_audit_events_monthly"=>0,
|
||||
"compliance_total_unique_counts_weekly"=>0,
|
||||
"compliance_total_unique_counts_monthly"=>0},
|
||||
"analytics"=>
|
||||
{"users_viewing_analytics_group_devops_adoption_weekly"=>0,
|
||||
"users_viewing_analytics_group_devops_adoption_monthly"=>0,
|
||||
"analytics_total_unique_counts_weekly"=>0,
|
||||
"analytics_total_unique_counts_monthly"=>0},
|
||||
"ide_edit"=>
|
||||
{"users_editing_by_web_ide_weekly"=>0,
|
||||
"users_editing_by_web_ide_monthly"=>0,
|
||||
"users_editing_by_sfe_weekly"=>0,
|
||||
"users_editing_by_sfe_monthly"=>0,
|
||||
"ide_edit_total_unique_counts_weekly"=>0,
|
||||
"ide_edit_total_unique_counts_monthly"=>0}
|
||||
}
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
# Redis Counters
|
||||
redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
|
||||
redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
|
||||
|
||||
# Define events in common.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml
|
||||
|
||||
# Tracking events
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_expanding_vulnerabilities', values: visitor_id)
|
||||
|
||||
# Get unique events for metric
|
||||
redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) }
|
||||
```
|
||||
|
||||
### Alternative counters
|
||||
|
||||
Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception.
|
||||
Mainly used for settings and configurations.
|
||||
|
||||
Method:
|
||||
|
||||
```ruby
|
||||
alt_usage_data(value = nil, fallback: -1, &block)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `value`: a simple static value in which case the value is simply returned.
|
||||
- or a `block`: which is evaluated
|
||||
- `fallback: -1`: the common value used for any metrics that are failing.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
alt_usage_data { Gitlab::VERSION }
|
||||
alt_usage_data { Gitlab::CurrentSettings.uuid }
|
||||
alt_usage_data(999)
|
||||
```
|
||||
|
||||
### Add counters to build new metrics
|
||||
|
||||
When adding the results of two counters, use the `add` Service Data method that
|
||||
handles fallback values and exceptions. It also generates a valid [SQL export](index.md#export-service-ping-sql-queries-and-definitions).
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
add(User.active, User.bot)
|
||||
```
|
||||
|
||||
### Prometheus queries
|
||||
|
||||
In those cases where operational metrics should be part of Service Ping, a database or Redis query is unlikely
|
||||
to provide useful data. Instead, Prometheus might be more appropriate, because most GitLab architectural
|
||||
components publish metrics to it that can be queried back, aggregated, and included as Service Data.
|
||||
|
||||
NOTE:
|
||||
Prometheus as a data source for Service Ping is only available for single-node Omnibus installations
|
||||
that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance.
|
||||
|
||||
To query Prometheus for metrics, a helper method is available to `yield` a fully configured
|
||||
`PrometheusClient`, given it is available as per the note above:
|
||||
|
||||
```ruby
|
||||
with_prometheus_client do |client|
|
||||
response = client.query('<your query>')
|
||||
...
|
||||
end
|
||||
```
|
||||
|
||||
Refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb)
|
||||
for how to use its API to query for data.
|
||||
|
||||
### Fallback values for Service Ping
|
||||
|
||||
We return fallback values in these cases:
|
||||
|
||||
| Case | Value |
|
||||
|-----------------------------|-------|
|
||||
| Deprecated Metric | -1000 |
|
||||
| Timeouts, general failures | -1 |
|
||||
| Standard errors in counters | -2 |
|
||||
|
||||
## Name and place the metric
|
||||
|
||||
Add the metric in one of the top-level keys:
|
||||
|
|
@ -159,7 +819,7 @@ To set up Service Ping locally, you must:
|
|||
|
||||
## Test Prometheus-based Service Ping
|
||||
|
||||
If the data submitted includes metrics [queried from Prometheus](index.md#prometheus-queries)
|
||||
If the data submitted includes metrics [queried from Prometheus](#prometheus-queries)
|
||||
you want to inspect and verify, you must:
|
||||
|
||||
- Ensure that a Prometheus server is running locally.
|
||||
|
|
@ -208,3 +868,197 @@ However, it has the following limitations:
|
|||
with any of the other running services. That is not how node metrics are reported in a production setup, where `node_exporter`
|
||||
always runs as a process alongside other GitLab components on any given node. For Service Ping, none of the node data would therefore
|
||||
appear to be associated to any of the services running, because they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics appears in Service Ping.
|
||||
|
||||
## Aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6.
|
||||
|
||||
WARNING:
|
||||
This feature is intended solely for internal GitLab use.
|
||||
|
||||
To add data for aggregated metrics to the Service Ping payload, add a corresponding definition to:
|
||||
|
||||
- [`config/metrics/aggregates/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/aggregates/) for metrics available in the Community Edition.
|
||||
- [`ee/config/metrics/aggregates/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/aggregates/) for metrics available in the Enterprise Edition.
|
||||
|
||||
Each aggregate definition includes following parts:
|
||||
|
||||
- `name`: Unique name under which the aggregate metric is added to the Service Ping payload.
|
||||
- `operator`: Operator that defines how the aggregated metric data is counted. Available operators are:
|
||||
- `OR`: Removes duplicates and counts all entries that triggered any of listed events.
|
||||
- `AND`: Removes duplicates and counts all elements that were observed triggering all of following events.
|
||||
- `time_frame`: One or more valid time frames. Use these to limit the data included in aggregated metric to events within a specific date-range. Valid time frames are:
|
||||
- `7d`: Last seven days of data.
|
||||
- `28d`: Last twenty eight days of data.
|
||||
- `all`: All historical data, only available for `database` sourced aggregated metrics.
|
||||
- `source`: Data source used to collect all events data included in aggregated metric. Valid data sources are:
|
||||
- [`database`](#database-sourced-aggregated-metrics)
|
||||
- [`redis`](#redis-sourced-aggregated-metrics)
|
||||
- `events`: list of events names to aggregate into metric. All events in this list must
|
||||
relay on the same data source. Additional data source requirements are described in the
|
||||
[Database sourced aggregated metrics](#database-sourced-aggregated-metrics) and
|
||||
[Redis sourced aggregated metrics](#redis-sourced-aggregated-metrics) sections.
|
||||
- `feature_flag`: Name of [development feature flag](../feature_flags/index.md#development-type)
|
||||
that is checked before metrics aggregation is performed. Corresponding feature flag
|
||||
should have `default_enabled` attribute set to `false`. The `feature_flag` attribute
|
||||
is optional and can be omitted. When `feature_flag` is missing, no feature flag is checked.
|
||||
|
||||
Example aggregated metric entries:
|
||||
|
||||
```yaml
|
||||
- name: example_metrics_union
|
||||
operator: OR
|
||||
events:
|
||||
- 'users_expanding_secure_security_report'
|
||||
- 'users_expanding_testing_code_quality_report'
|
||||
- 'users_expanding_testing_accessibility_report'
|
||||
source: redis
|
||||
time_frame:
|
||||
- 7d
|
||||
- 28d
|
||||
- name: example_metrics_intersection
|
||||
operator: AND
|
||||
source: database
|
||||
time_frame:
|
||||
- 28d
|
||||
- all
|
||||
events:
|
||||
- 'dependency_scanning_pipeline_all_time'
|
||||
- 'container_scanning_pipeline_all_time'
|
||||
feature_flag: example_aggregated_metric
|
||||
```
|
||||
|
||||
Aggregated metrics collected in `7d` and `28d` time frames are added into Service Ping payload under the `aggregated_metrics` sub-key in the `counts_weekly` and `counts_monthly` top level keys.
|
||||
|
||||
```ruby
|
||||
{
|
||||
:counts_monthly => {
|
||||
:deployments => 1003,
|
||||
:successful_deployments => 78,
|
||||
:failed_deployments => 275,
|
||||
:packages => 155,
|
||||
:personal_snippets => 2106,
|
||||
:project_snippets => 407,
|
||||
:promoted_issues => 719,
|
||||
:aggregated_metrics => {
|
||||
:example_metrics_union => 7,
|
||||
:example_metrics_intersection => 2
|
||||
},
|
||||
:snippets => 2513
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Aggregated metrics for `all` time frame are present in the `count` top level key, with the `aggregate_` prefix added to their name.
|
||||
|
||||
For example:
|
||||
|
||||
`example_metrics_intersection`
|
||||
|
||||
Becomes:
|
||||
|
||||
`counts.aggregate_example_metrics_intersection`
|
||||
|
||||
```ruby
|
||||
{
|
||||
:counts => {
|
||||
:deployments => 11003,
|
||||
:successful_deployments => 178,
|
||||
:failed_deployments => 1275,
|
||||
:aggregate_example_metrics_intersection => 12
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Redis sourced aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6.
|
||||
|
||||
To declare the aggregate of events collected with [Redis HLL Counters](#redis-hll-counters),
|
||||
you must fulfill the following requirements:
|
||||
|
||||
1. All events listed at `events` attribute must come from
|
||||
[`known_events/*.yml`](#known-events-are-added-automatically-in-service-data-payload) files.
|
||||
1. All events listed at `events` attribute must have the same `redis_slot` attribute.
|
||||
1. All events listed at `events` attribute must have the same `aggregation` attribute.
|
||||
1. `time_frame` does not include `all` value, which is unavailable for Redis sourced aggregated metrics.
|
||||
|
||||
### Database sourced aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52784) in GitLab 13.9.
|
||||
> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
|
||||
> - It's enabled on GitLab.com.
|
||||
|
||||
To declare an aggregate of metrics based on events collected from database, follow
|
||||
these steps:
|
||||
|
||||
1. [Persist the metrics for aggregation](#persist-metrics-for-aggregation).
|
||||
1. [Add new aggregated metric definition](#add-new-aggregated-metric-definition).
|
||||
|
||||
#### Persist metrics for aggregation
|
||||
|
||||
Only metrics calculated with [Estimated Batch Counters](#estimated-batch-counters)
|
||||
can be persisted for database sourced aggregated metrics. To persist a metric,
|
||||
inject a Ruby block into the
|
||||
[estimate_batch_distinct_count](#estimate_batch_distinct_count-method) method.
|
||||
This block should invoke the
|
||||
`Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll.save_aggregated_metrics`
|
||||
[method](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb#L21),
|
||||
which stores `estimate_batch_distinct_count` results for future use in aggregated metrics.
|
||||
|
||||
The `Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll.save_aggregated_metrics`
|
||||
method accepts the following arguments:
|
||||
|
||||
- `metric_name`: The name of metric to use for aggregations. Should be the same
|
||||
as the key under which the metric is added into Service Ping.
|
||||
- `recorded_at_timestamp`: The timestamp representing the moment when a given
|
||||
Service Ping payload was collected. You should use the convenience method `recorded_at`
|
||||
to fill `recorded_at_timestamp` argument, like this: `recorded_at_timestamp: recorded_at`
|
||||
- `time_period`: The time period used to build the `relation` argument passed into
|
||||
`estimate_batch_distinct_count`. To collect the metric with all available historical
|
||||
data, set a `nil` value as time period: `time_period: nil`.
|
||||
- `data`: HyperLogLog buckets structure representing unique entries in `relation`.
|
||||
The `estimate_batch_distinct_count` method always passes the correct argument
|
||||
into the block, so `data` argument must always have a value equal to block argument,
|
||||
like this: `data: result`
|
||||
|
||||
Example metrics persistence:
|
||||
|
||||
```ruby
|
||||
class UsageData
|
||||
def count_secure_pipelines(time_period)
|
||||
...
|
||||
relation = ::Security::Scan.latest_successful_by_build.by_scan_types(scan_type).where(security_scans: time_period)
|
||||
|
||||
pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
|
||||
::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
|
||||
.save_aggregated_metrics(metric_name: 'dependency_scanning_pipeline', recorded_at_timestamp: recorded_at, time_period: time_period, data: result)
|
||||
end
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
#### Add new aggregated metric definition
|
||||
|
||||
After all metrics are persisted, you can add an aggregated metric definition at
|
||||
[`aggregated_metrics/`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/aggregates/).
|
||||
|
||||
To declare the aggregate of metrics collected with [Estimated Batch Counters](#estimated-batch-counters),
|
||||
you must fulfill the following requirements:
|
||||
|
||||
- Metrics names listed in the `events:` attribute, have to use the same names you passed in the `metric_name` argument while persisting metrics in previous step.
|
||||
- Every metric listed in the `events:` attribute, has to be persisted for **every** selected `time_frame:` value.
|
||||
|
||||
Example definition:
|
||||
|
||||
```yaml
|
||||
- name: example_metrics_intersection_database_sourced
|
||||
operator: AND
|
||||
source: database
|
||||
events:
|
||||
- 'dependency_scanning_pipeline'
|
||||
- 'container_scanning_pipeline'
|
||||
time_frame:
|
||||
- 28d
|
||||
- all
|
||||
```
|
||||
|
|
|
|||
|
|
@ -222,844 +222,7 @@ We also collect metrics specific to [Geo](../../administration/geo/index.md) sec
|
|||
|
||||
## Implementing Service Ping
|
||||
|
||||
Service Ping consists of two kinds of data, counters and observations. Counters track how often a certain event
|
||||
happened over time, such as how many CI pipelines have run. They are monotonic and always trend up.
|
||||
Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no
|
||||
general guidelines around how to collect those, due to the individual nature of that data.
|
||||
|
||||
### Types of counters
|
||||
|
||||
There are several types of counters in `usage_data.rb`:
|
||||
|
||||
- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation
|
||||
- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation in a given column
|
||||
- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation in a given column
|
||||
- **Alternative Counters:** Used for settings and configurations
|
||||
- **Redis Counters:** Used for in-memory counts.
|
||||
|
||||
NOTE:
|
||||
Only use the provided counter methods. Each counter method contains a built-in fail-safe mechanism that isolates each counter to avoid breaking the entire Service Ping process.
|
||||
|
||||
### Instrumentation classes
|
||||
|
||||
We recommend you use [instrumentation classes](metrics_instrumentation.md) in `usage_data.rb` where possible.
|
||||
|
||||
For example, we have the following instrumentation class:
|
||||
`lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb`.
|
||||
|
||||
You should add it to `usage_data.rb` as follows:
|
||||
|
||||
```ruby
|
||||
boards: add_metric('CountBoardsMetric', time_frame: 'all'),
|
||||
```
|
||||
|
||||
### Batch counting
|
||||
|
||||
For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query.
|
||||
|
||||
For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables:
|
||||
|
||||
| Table | Row counts in millions |
|
||||
|------------------------------|------------------------|
|
||||
| `merge_request_diff_commits` | 2280 |
|
||||
| `ci_build_trace_sections` | 1764 |
|
||||
| `merge_request_diff_files` | 1082 |
|
||||
| `events` | 514 |
|
||||
|
||||
The following operation methods are available:
|
||||
|
||||
- [Ordinary batch counters](#ordinary-batch-counters)
|
||||
- [Distinct batch counters](#distinct-batch-counters)
|
||||
- [Sum batch operation](#sum-batch-operation)
|
||||
- [Add operation](#add-operation)
|
||||
- [Estimated batch counters](#estimated-batch-counters)
|
||||
|
||||
Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases,
|
||||
you may need to add a specialized index on the columns involved in a counter.
|
||||
|
||||
### Ordinary batch counters
|
||||
|
||||
Handles `ActiveRecord::StatementInvalid` error
|
||||
|
||||
Simple count of a given `ActiveRecord_Relation`, does a non-distinct batch count, smartly reduces `batch_size`, and handles errors.
|
||||
|
||||
Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation` the ActiveRecord_Relation to perform the count
|
||||
- `column` the column to perform the count on, by default is the primary key
|
||||
- `batch`: default `true` to use batch counting
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
count(User.active)
|
||||
count(::Clusters::Cluster.aws_installed.enabled, :cluster_id)
|
||||
count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id))
|
||||
```
|
||||
|
||||
### Distinct batch counters
|
||||
|
||||
Handles `ActiveRecord::StatementInvalid` error
|
||||
|
||||
Distinct count of a given `ActiveRecord_Relation` on given column, a distinct batch count, smartly reduces `batch_size`, and handles errors.
|
||||
|
||||
Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation` the ActiveRecord_Relation to perform the count
|
||||
- `column` the column to perform the distinct count, by default is the primary key
|
||||
- `batch`: default `true` to use batch counting
|
||||
- `batch_size`: if none set it uses default value 10000 from `Gitlab::Database::BatchCounter`
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
WARNING:
|
||||
Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../iterating_tables_in_batches.md) guide.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
distinct_count(::Project, :creator_id)
|
||||
distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id))
|
||||
distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id')
|
||||
```
|
||||
|
||||
### Sum batch operation
|
||||
|
||||
Handles `ActiveRecord::StatementInvalid` error
|
||||
|
||||
Sum the values of a given ActiveRecord_Relation on given column and handles errors.
|
||||
|
||||
Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `relation` the ActiveRecord_Relation to perform the operation
|
||||
- `column` the column to sum on
|
||||
- `batch_size`: if none set it uses default value 1000 from `Gitlab::Database::BatchCounter`
|
||||
- `start`: custom start of the batch counting to avoid complex min calculations
|
||||
- `end`: custom end of the batch counting to avoid complex min calculations
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
sum(JiraImportState.finished, :imported_issues_count)
|
||||
```
|
||||
|
||||
### Grouping and batch operations
|
||||
|
||||
The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation`
|
||||
object, which groups by a specified column. With a grouped relation, the methods do batch counting,
|
||||
handle errors, and returns a hash table of key-value pairs.
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
count(Namespace.group(:type))
|
||||
# returns => {nil=>179, "Group"=>54}
|
||||
|
||||
distinct_count(Project.group(:visibility_level), :creator_id)
|
||||
# returns => {0=>1, 10=>1, 20=>11}
|
||||
|
||||
sum(Issue.group(:state_id), :weight))
|
||||
# returns => {1=>3542, 2=>6820}
|
||||
```
|
||||
|
||||
### Add operation
|
||||
|
||||
Handles `StandardError`.
|
||||
|
||||
Returns `-1` if any of the arguments are `-1`.
|
||||
|
||||
Sum the values given as parameters.
|
||||
|
||||
Method: `add(*args)`
|
||||
|
||||
Examples:
|
||||
|
||||
```ruby
|
||||
project_imports = distinct_count(::Project.where.not(import_type: nil), :creator_id)
|
||||
bulk_imports = distinct_count(::BulkImport, :user_id)
|
||||
|
||||
add(project_imports, bulk_imports)
|
||||
```
|
||||
|
||||
### Estimated batch counters
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48233) in GitLab 13.7.
|
||||
|
||||
Estimated batch counter functionality handles `ActiveRecord::StatementInvalid` errors
|
||||
when used through the provided `estimate_batch_distinct_count` method.
|
||||
Errors return a value of `-1`.
|
||||
|
||||
WARNING:
|
||||
This functionality estimates a distinct count of a specific ActiveRecord_Relation in a given column,
|
||||
which uses the [HyperLogLog](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) algorithm.
|
||||
As the HyperLogLog algorithm is probabilistic, the **results always include error**.
|
||||
The highest encountered error rate is 4.9%.
|
||||
|
||||
When correctly used, the `estimate_batch_distinct_count` method enables efficient counting over
|
||||
columns that contain non-unique values, which can not be assured by other counters.
|
||||
|
||||
#### estimate_batch_distinct_count method
|
||||
|
||||
Method: `estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)`
|
||||
|
||||
The [method](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/utils/usage_data.rb#L63)
|
||||
includes the following arguments:
|
||||
|
||||
- `relation`: The ActiveRecord_Relation to perform the count.
|
||||
- `column`: The column to perform the distinct count. The default is the primary key.
|
||||
- `batch_size`: From `Gitlab::Database::PostgresHll::BatchDistinctCounter::DEFAULT_BATCH_SIZE`. Default value: 10,000.
|
||||
- `start`: The custom start of the batch count, to avoid complex minimum calculations.
|
||||
- `finish`: The custom end of the batch count to avoid complex maximum calculations.
|
||||
|
||||
The method includes the following prerequisites:
|
||||
|
||||
1. The supplied `relation` must include the primary key defined as the numeric column.
|
||||
For example: `id bigint NOT NULL`.
|
||||
1. The `estimate_batch_distinct_count` can handle a joined relation. To use its ability to
|
||||
count non-unique columns, the joined relation **must not** have a one-to-many relationship,
|
||||
such as `has_many :boards`.
|
||||
1. Both `start` and `finish` arguments should always represent primary key relationship values,
|
||||
even if the estimated count refers to another column, for example:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Note, :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id))
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
1. Simple execution of estimated batch counter, with only relation provided,
|
||||
returned value represents estimated number of unique values in `id` column
|
||||
(which is the primary key) of `Project` relation:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Project)
|
||||
```
|
||||
|
||||
1. Execution of estimated batch counter, where provided relation has applied
|
||||
additional filter (`.where(time_period)`), number of unique values estimated
|
||||
in custom column (`:author_id`), and parameters: `start` and `finish` together
|
||||
apply boundaries that defines range of provided relation to analyze:
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id))
|
||||
```
|
||||
|
||||
1. Execution of estimated batch counter with joined relation (`joins(:cluster)`),
|
||||
for a custom column (`'clusters.user_id'`):
|
||||
|
||||
```ruby
|
||||
estimate_batch_distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id')
|
||||
```
|
||||
|
||||
When instrumenting metric with usage of estimated batch counter please add
|
||||
`_estimated` suffix to its name, for example:
|
||||
|
||||
```ruby
|
||||
"counts": {
|
||||
"ci_builds_estimated": estimate_batch_distinct_count(Ci::Build),
|
||||
...
|
||||
```
|
||||
|
||||
### Redis counters
|
||||
|
||||
Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent`
|
||||
returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent
|
||||
different behavior due to 2 different implementations of Redis counter
|
||||
|
||||
Method: `redis_usage_data(counter, &block)`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented
|
||||
- or a `block`: which is evaluated
|
||||
|
||||
#### Ordinary Redis counters
|
||||
|
||||
Examples of implementation:
|
||||
|
||||
- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb)
|
||||
- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb)
|
||||
|
||||
##### UsageData API tracking
|
||||
|
||||
<!-- There's nearly identical content in `##### Adding new events`. If you fix errors here, you may need to fix the same errors in the other location. -->
|
||||
|
||||
1. Track event using `UsageData` API
|
||||
|
||||
Increment event count using ordinary Redis counter, for given event name.
|
||||
|
||||
Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default.
|
||||
|
||||
API requests are protected by checking for a valid CSRF token.
|
||||
|
||||
To be able to increment the values, the related feature `usage_data_<event_name>` should be enabled.
|
||||
|
||||
```plaintext
|
||||
POST /usage_data/increment_counter
|
||||
```
|
||||
|
||||
| Attribute | Type | Required | Description |
|
||||
| :-------- | :--- | :------- | :---------- |
|
||||
| `event` | string | yes | The event name it should be tracked |
|
||||
|
||||
Response:
|
||||
|
||||
- `200` if event was tracked
|
||||
- `400 Bad request` if event parameter is missing
|
||||
- `401 Unauthorized` if user is not authenticated
|
||||
- `403 Forbidden` for invalid CSRF token provided
|
||||
|
||||
1. Track events using JavaScript/Vue API helper which calls the API above
|
||||
|
||||
Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled to be able to track events
|
||||
|
||||
```javascript
|
||||
import api from '~/api';
|
||||
|
||||
api.trackRedisCounterEvent('my_already_defined_event_name'),
|
||||
```
|
||||
|
||||
#### Redis HLL counters
|
||||
|
||||
WARNING:
|
||||
HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes some small error**. According to [Redis documentation](https://redis.io/commands/pfcount), data from
|
||||
used HLL implementation is "approximated with a standard error of 0.81%".
|
||||
|
||||
With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values.
|
||||
|
||||
Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount).
|
||||
|
||||
##### Add new events
|
||||
|
||||
1. Define events in [`known_events`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/).
|
||||
|
||||
Example event:
|
||||
|
||||
```yaml
|
||||
- name: users_creating_epics
|
||||
category: epics_usage
|
||||
redis_slot: users
|
||||
aggregation: weekly
|
||||
feature_flag: track_epics_activity
|
||||
```
|
||||
|
||||
Keys:
|
||||
|
||||
- `name`: unique event name.
|
||||
|
||||
Name format for Redis HLL events `<name>_<redis_slot>`.
|
||||
|
||||
[See Metric name](metrics_dictionary.md#metric-name) for a complete guide on metric naming suggestion.
|
||||
|
||||
Consider including in the event's name the Redis slot to be able to count totals for a specific category.
|
||||
|
||||
Example names: `users_creating_epics`, `users_triggering_security_scans`.
|
||||
|
||||
- `category`: event category. Used for getting total counts for events in a category, for easier
|
||||
access to a group of events.
|
||||
- `redis_slot`: optional Redis slot. Default value: event name. Only event data that is stored in the same slot
|
||||
can be aggregated. Ensure keys are in the same slot. For example:
|
||||
`users_creating_epics` with `redis_slot: 'users'` builds Redis key
|
||||
`{users}_creating_epics-2020-34`. If `redis_slot` is not defined the Redis key will
|
||||
be `{users_creating_epics}-2020-34`.
|
||||
Recommended slots to use are: `users`, `projects`. This is the value we count.
|
||||
- `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly
|
||||
aggregation.
|
||||
- `aggregation`: may be set to a `:daily` or `:weekly` key. Defines how counting data is stored in Redis.
|
||||
Aggregation on a `daily` basis does not pull more fine grained data.
|
||||
- `feature_flag`: optional `default_enabled: :yaml`. If no feature flag is set then the tracking is enabled. One feature flag can be used for multiple events. For details, see our [GitLab internal Feature flags](../feature_flags/index.md) documentation. The feature flags are owned by the group adding the event tracking.
|
||||
|
||||
1. Use one of the following methods to track the event:
|
||||
|
||||
- In the controller using the `RedisTracking` module and the following format:
|
||||
|
||||
```ruby
|
||||
track_redis_hll_event(*controller_actions, name:, if: nil, &block)
|
||||
```
|
||||
|
||||
Arguments:
|
||||
|
||||
- `controller_actions`: the controller actions to track.
|
||||
- `name`: the event name.
|
||||
- `if`: optional custom conditions. Uses the same format as Rails callbacks.
|
||||
- `&block`: optional block that computes and returns the `custom_id` that we want to track. This overrides the `visitor_id`.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
# controller
|
||||
class ProjectsController < Projects::ApplicationController
|
||||
include RedisTracking
|
||||
|
||||
skip_before_action :authenticate_user!, only: :show
|
||||
track_redis_hll_event :index, :show, name: 'users_visiting_projects'
|
||||
|
||||
def index
|
||||
render html: 'index'
|
||||
end
|
||||
|
||||
def new
|
||||
render html: 'new'
|
||||
end
|
||||
|
||||
def show
|
||||
render html: 'show'
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
- In the API using the `increment_unique_values(event_name, values)` helper method.
|
||||
|
||||
Arguments:
|
||||
|
||||
- `event_name`: the event name.
|
||||
- `values`: the values counted. Can be one value or an array of values.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
get ':id/registry/repositories' do
|
||||
repositories = ContainerRepositoriesFinder.new(
|
||||
user: current_user, subject: user_group
|
||||
).execute
|
||||
|
||||
increment_unique_values('users_listing_repositories', current_user.id)
|
||||
|
||||
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
|
||||
end
|
||||
```
|
||||
|
||||
- Using `track_usage_event(event_name, values)` in services and GraphQL.
|
||||
|
||||
Increment unique values count using Redis HLL, for a given event name.
|
||||
|
||||
Examples:
|
||||
|
||||
- [Track usage event for an incident in a service](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/app/services/issues/update_service.rb#L66)
|
||||
- [Track usage event for an incident in GraphQL](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/app/graphql/mutations/alert_management/update_alert_status.rb#L16)
|
||||
|
||||
```ruby
|
||||
track_usage_event(:incident_management_incident_created, current_user.id)
|
||||
```
|
||||
|
||||
- Using the `UsageData` API.
|
||||
<!-- There's nearly identical content in `##### UsageData API Tracking`. If you find / fix errors here, you may need to fix errors in that section too. -->
|
||||
|
||||
Increment unique users count using Redis HLL, for a given event name.
|
||||
|
||||
To track events using the `UsageData` API, ensure the `usage_data_api` feature flag
|
||||
is set to `default_enabled: true`. Enabled by default in GitLab 13.7 and later.
|
||||
|
||||
API requests are protected by checking for a valid CSRF token.
|
||||
|
||||
```plaintext
|
||||
POST /usage_data/increment_unique_users
|
||||
```
|
||||
|
||||
| Attribute | Type | Required | Description |
|
||||
| :-------- | :--- | :------- | :---------- |
|
||||
| `event` | string | yes | The event name to track |
|
||||
|
||||
Response:
|
||||
|
||||
- `200` if the event was tracked, or if tracking failed for any reason.
|
||||
- `400 Bad request` if an event parameter is missing.
|
||||
- `401 Unauthorized` if the user is not authenticated.
|
||||
- `403 Forbidden` if an invalid CSRF token is provided.
|
||||
|
||||
- Using the JavaScript/Vue API helper, which calls the `UsageData` API.
|
||||
|
||||
To track events using the `UsageData` API, ensure the `usage_data_api` feature flag
|
||||
is set to `default_enabled: true`. Enabled by default in GitLab 13.7 and later.
|
||||
|
||||
Example for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/):
|
||||
|
||||
```javascript
|
||||
import api from '~/api';
|
||||
|
||||
api.trackRedisHllUserEvent('my_already_defined_event_name'),
|
||||
```
|
||||
|
||||
1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date:, context: '')`.
|
||||
|
||||
Arguments:
|
||||
|
||||
- `event_names`: the list of event names.
|
||||
- `start_date`: start date of the period for which we want to get event data.
|
||||
- `end_date`: end date of the period for which we want to get event data.
|
||||
- `context`: context of the event. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate`.
|
||||
|
||||
1. Testing tracking and getting unique events
|
||||
|
||||
Trigger events in rails console by using `track_event` method
|
||||
|
||||
```ruby
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_viewing_compliance_audit_events', values: 1)
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_viewing_compliance_audit_events', values: [2, 3])
|
||||
```
|
||||
|
||||
Next, get the unique events for the current week.
|
||||
|
||||
```ruby
|
||||
# Get unique events for metric for current_week
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_viewing_compliance_audit_events',
|
||||
start_date: Date.current.beginning_of_week, end_date: Date.current.next_week)
|
||||
```
|
||||
|
||||
##### Recommendations
|
||||
|
||||
We have the following recommendations for [adding new events](#add-new-events):
|
||||
|
||||
- Event aggregation: weekly.
|
||||
- Key expiry time:
|
||||
- Daily: 29 days.
|
||||
- Weekly: 42 days.
|
||||
- When adding new metrics, use a [feature flag](../../operations/feature_flags.md) to control the impact.
|
||||
- For feature flags triggered by another service, set `default_enabled: false`,
|
||||
- Events can be triggered using the `UsageData` API, which helps when there are > 10 events per change
|
||||
|
||||
##### Enable or disable Redis HLL tracking
|
||||
|
||||
Events are tracked behind optional [feature flags](../feature_flags/index.md) due to concerns for Redis performance and scalability.
|
||||
|
||||
For a full list of events and corresponding feature flags see, [known_events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files.
|
||||
|
||||
To enable or disable tracking for specific event in <https://gitlab.com> or <https://about.staging.gitlab.com>, run commands such as the following to
|
||||
[enable or disable the corresponding feature](../feature_flags/index.md).
|
||||
|
||||
```shell
|
||||
/chatops run feature set <feature_name> true
|
||||
/chatops run feature set <feature_name> false
|
||||
```
|
||||
|
||||
We can also disable tracking completely by using the global flag:
|
||||
|
||||
```shell
|
||||
/chatops run feature set redis_hll_tracking true
|
||||
/chatops run feature set redis_hll_tracking false
|
||||
```
|
||||
|
||||
##### Known events are added automatically in Service Data payload
|
||||
|
||||
All events added in [`known_events/common.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml) are automatically added to Service Data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209).
|
||||
For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable:
|
||||
|
||||
- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#add-new-events) events and data for the last complete week for weekly [aggregation](#add-new-events) events.
|
||||
- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#add-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#add-new-events) events.
|
||||
|
||||
Redis HLL implementation calculates automatic total metrics, if there are more than one metric for the same category, aggregation, and Redis slot.
|
||||
|
||||
- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric.
|
||||
- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric.
|
||||
|
||||
Example of `redis_hll_counters` data:
|
||||
|
||||
```ruby
|
||||
{:redis_hll_counters=>
|
||||
{"compliance"=>
|
||||
{"users_viewing_compliance_dashboard_weekly"=>0,
|
||||
"users_viewing_compliance_dashboard_monthly"=>0,
|
||||
"users_viewing_compliance_audit_events_weekly"=>0,
|
||||
"users_viewing_audit_events_monthly"=>0,
|
||||
"compliance_total_unique_counts_weekly"=>0,
|
||||
"compliance_total_unique_counts_monthly"=>0},
|
||||
"analytics"=>
|
||||
{"users_viewing_analytics_group_devops_adoption_weekly"=>0,
|
||||
"users_viewing_analytics_group_devops_adoption_monthly"=>0,
|
||||
"analytics_total_unique_counts_weekly"=>0,
|
||||
"analytics_total_unique_counts_monthly"=>0},
|
||||
"ide_edit"=>
|
||||
{"users_editing_by_web_ide_weekly"=>0,
|
||||
"users_editing_by_web_ide_monthly"=>0,
|
||||
"users_editing_by_sfe_weekly"=>0,
|
||||
"users_editing_by_sfe_monthly"=>0,
|
||||
"ide_edit_total_unique_counts_weekly"=>0,
|
||||
"ide_edit_total_unique_counts_monthly"=>0}
|
||||
}
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
# Redis Counters
|
||||
redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
|
||||
redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
|
||||
|
||||
# Define events in common.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml
|
||||
|
||||
# Tracking events
|
||||
Gitlab::UsageDataCounters::HLLRedisCounter.track_event('users_expanding_vulnerabilities', values: visitor_id)
|
||||
|
||||
# Get unique events for metric
|
||||
redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) }
|
||||
```
|
||||
|
||||
### Alternative counters
|
||||
|
||||
Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception.
|
||||
Mainly used for settings and configurations.
|
||||
|
||||
Method: `alt_usage_data(value = nil, fallback: -1, &block)`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `value`: a simple static value in which case the value is simply returned.
|
||||
- or a `block`: which is evaluated
|
||||
- `fallback: -1`: the common value used for any metrics that are failing.
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
alt_usage_data { Gitlab::VERSION }
|
||||
alt_usage_data { Gitlab::CurrentSettings.uuid }
|
||||
alt_usage_data(999)
|
||||
```
|
||||
|
||||
### Add counters to build new metrics
|
||||
|
||||
When adding the results of two counters, use the `add` Service Data method that
|
||||
handles fallback values and exceptions. It also generates a valid [SQL export](#export-service-ping-sql-queries-and-definitions).
|
||||
|
||||
Example:
|
||||
|
||||
```ruby
|
||||
add(User.active, User.bot)
|
||||
```
|
||||
|
||||
### Prometheus queries
|
||||
|
||||
In those cases where operational metrics should be part of Service Ping, a database or Redis query is unlikely
|
||||
to provide useful data. Instead, Prometheus might be more appropriate, because most GitLab architectural
|
||||
components publish metrics to it that can be queried back, aggregated, and included as Service Data.
|
||||
|
||||
NOTE:
|
||||
Prometheus as a data source for Service Ping is only available for single-node Omnibus installations
|
||||
that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance.
|
||||
|
||||
To query Prometheus for metrics, a helper method is available to `yield` a fully configured
|
||||
`PrometheusClient`, given it is available as per the note above:
|
||||
|
||||
```ruby
|
||||
with_prometheus_client do |client|
|
||||
response = client.query('<your query>')
|
||||
...
|
||||
end
|
||||
```
|
||||
|
||||
Refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb)
|
||||
for how to use its API to query for data.
|
||||
|
||||
### Fallback values for Service Ping
|
||||
|
||||
We return fallback values in these cases:
|
||||
|
||||
| Case | Value |
|
||||
|-----------------------------|-------|
|
||||
| Deprecated Metric | -1000 |
|
||||
| Timeouts, general failures | -1 |
|
||||
| Standard errors in counters | -2 |
|
||||
|
||||
## Aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6.
|
||||
|
||||
WARNING:
|
||||
This feature is intended solely for internal GitLab use.
|
||||
|
||||
To add data for aggregated metrics to the Service Ping payload, add a corresponding definition to:
|
||||
|
||||
- [`config/metrics/aggregates/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/aggregates/) for metrics available in the Community Edition.
|
||||
- [`ee/config/metrics/aggregates/*.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/aggregates/) for metrics available in the Enterprise Edition.
|
||||
|
||||
Each aggregate definition includes following parts:
|
||||
|
||||
- `name`: Unique name under which the aggregate metric is added to the Service Ping payload.
|
||||
- `operator`: Operator that defines how the aggregated metric data is counted. Available operators are:
|
||||
- `OR`: Removes duplicates and counts all entries that triggered any of listed events.
|
||||
- `AND`: Removes duplicates and counts all elements that were observed triggering all of following events.
|
||||
- `time_frame`: One or more valid time frames. Use these to limit the data included in aggregated metric to events within a specific date-range. Valid time frames are:
|
||||
- `7d`: Last seven days of data.
|
||||
- `28d`: Last twenty eight days of data.
|
||||
- `all`: All historical data, only available for `database` sourced aggregated metrics.
|
||||
- `source`: Data source used to collect all events data included in aggregated metric. Valid data sources are:
|
||||
- [`database`](#database-sourced-aggregated-metrics)
|
||||
- [`redis`](#redis-sourced-aggregated-metrics)
|
||||
- `events`: list of events names to aggregate into metric. All events in this list must
|
||||
relay on the same data source. Additional data source requirements are described in the
|
||||
[Database sourced aggregated metrics](#database-sourced-aggregated-metrics) and
|
||||
[Redis sourced aggregated metrics](#redis-sourced-aggregated-metrics) sections.
|
||||
- `feature_flag`: Name of [development feature flag](../feature_flags/index.md#development-type)
|
||||
that is checked before metrics aggregation is performed. Corresponding feature flag
|
||||
should have `default_enabled` attribute set to `false`. The `feature_flag` attribute
|
||||
is optional and can be omitted. When `feature_flag` is missing, no feature flag is checked.
|
||||
|
||||
Example aggregated metric entries:
|
||||
|
||||
```yaml
|
||||
- name: example_metrics_union
|
||||
operator: OR
|
||||
events:
|
||||
- 'users_expanding_secure_security_report'
|
||||
- 'users_expanding_testing_code_quality_report'
|
||||
- 'users_expanding_testing_accessibility_report'
|
||||
source: redis
|
||||
time_frame:
|
||||
- 7d
|
||||
- 28d
|
||||
- name: example_metrics_intersection
|
||||
operator: AND
|
||||
source: database
|
||||
time_frame:
|
||||
- 28d
|
||||
- all
|
||||
events:
|
||||
- 'dependency_scanning_pipeline_all_time'
|
||||
- 'container_scanning_pipeline_all_time'
|
||||
feature_flag: example_aggregated_metric
|
||||
```
|
||||
|
||||
Aggregated metrics collected in `7d` and `28d` time frames are added into Service Ping payload under the `aggregated_metrics` sub-key in the `counts_weekly` and `counts_monthly` top level keys.
|
||||
|
||||
```ruby
|
||||
{
|
||||
:counts_monthly => {
|
||||
:deployments => 1003,
|
||||
:successful_deployments => 78,
|
||||
:failed_deployments => 275,
|
||||
:packages => 155,
|
||||
:personal_snippets => 2106,
|
||||
:project_snippets => 407,
|
||||
:promoted_issues => 719,
|
||||
:aggregated_metrics => {
|
||||
:example_metrics_union => 7,
|
||||
:example_metrics_intersection => 2
|
||||
},
|
||||
:snippets => 2513
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Aggregated metrics for `all` time frame are present in the `count` top level key, with the `aggregate_` prefix added to their name.
|
||||
|
||||
For example:
|
||||
|
||||
`example_metrics_intersection`
|
||||
|
||||
Becomes:
|
||||
|
||||
`counts.aggregate_example_metrics_intersection`
|
||||
|
||||
```ruby
|
||||
{
|
||||
:counts => {
|
||||
:deployments => 11003,
|
||||
:successful_deployments => 178,
|
||||
:failed_deployments => 1275,
|
||||
:aggregate_example_metrics_intersection => 12
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Redis sourced aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6.
|
||||
|
||||
To declare the aggregate of events collected with [Redis HLL Counters](#redis-hll-counters),
|
||||
you must fulfill the following requirements:
|
||||
|
||||
1. All events listed at `events` attribute must come from
|
||||
[`known_events/*.yml`](#known-events-are-added-automatically-in-service-data-payload) files.
|
||||
1. All events listed at `events` attribute must have the same `redis_slot` attribute.
|
||||
1. All events listed at `events` attribute must have the same `aggregation` attribute.
|
||||
1. `time_frame` does not include `all` value, which is unavailable for Redis sourced aggregated metrics.
|
||||
|
||||
### Database sourced aggregated metrics
|
||||
|
||||
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52784) in GitLab 13.9.
|
||||
> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
|
||||
> - It's enabled on GitLab.com.
|
||||
|
||||
To declare an aggregate of metrics based on events collected from database, follow
|
||||
these steps:
|
||||
|
||||
1. [Persist the metrics for aggregation](#persist-metrics-for-aggregation).
|
||||
1. [Add new aggregated metric definition](#add-new-aggregated-metric-definition).
|
||||
|
||||
#### Persist metrics for aggregation
|
||||
|
||||
Only metrics calculated with [Estimated Batch Counters](#estimated-batch-counters)
|
||||
can be persisted for database sourced aggregated metrics. To persist a metric,
|
||||
inject a Ruby block into the
|
||||
[estimate_batch_distinct_count](#estimate_batch_distinct_count-method) method.
|
||||
This block should invoke the
|
||||
`Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll.save_aggregated_metrics`
|
||||
[method](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb#L21),
|
||||
which stores `estimate_batch_distinct_count` results for future use in aggregated metrics.
|
||||
|
||||
The `Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll.save_aggregated_metrics`
|
||||
method accepts the following arguments:
|
||||
|
||||
- `metric_name`: The name of metric to use for aggregations. Should be the same
|
||||
as the key under which the metric is added into Service Ping.
|
||||
- `recorded_at_timestamp`: The timestamp representing the moment when a given
|
||||
Service Ping payload was collected. You should use the convenience method `recorded_at`
|
||||
to fill `recorded_at_timestamp` argument, like this: `recorded_at_timestamp: recorded_at`
|
||||
- `time_period`: The time period used to build the `relation` argument passed into
|
||||
`estimate_batch_distinct_count`. To collect the metric with all available historical
|
||||
data, set a `nil` value as time period: `time_period: nil`.
|
||||
- `data`: HyperLogLog buckets structure representing unique entries in `relation`.
|
||||
The `estimate_batch_distinct_count` method always passes the correct argument
|
||||
into the block, so `data` argument must always have a value equal to block argument,
|
||||
like this: `data: result`
|
||||
|
||||
Example metrics persistence:
|
||||
|
||||
```ruby
|
||||
class UsageData
|
||||
def count_secure_pipelines(time_period)
|
||||
...
|
||||
relation = ::Security::Scan.latest_successful_by_build.by_scan_types(scan_type).where(security_scans: time_period)
|
||||
|
||||
pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
|
||||
::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
|
||||
.save_aggregated_metrics(metric_name: 'dependency_scanning_pipeline', recorded_at_timestamp: recorded_at, time_period: time_period, data: result)
|
||||
end
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
#### Add new aggregated metric definition
|
||||
|
||||
After all metrics are persisted, you can add an aggregated metric definition at
|
||||
[`aggregated_metrics/`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/aggregates/).
|
||||
|
||||
To declare the aggregate of metrics collected with [Estimated Batch Counters](#estimated-batch-counters),
|
||||
you must fulfill the following requirements:
|
||||
|
||||
- Metrics names listed in the `events:` attribute, have to use the same names you passed in the `metric_name` argument while persisting metrics in previous step.
|
||||
- Every metric listed in the `events:` attribute, has to be persisted for **every** selected `time_frame:` value.
|
||||
|
||||
Example definition:
|
||||
|
||||
```yaml
|
||||
- name: example_metrics_intersection_database_sourced
|
||||
operator: AND
|
||||
source: database
|
||||
events:
|
||||
- 'dependency_scanning_pipeline'
|
||||
- 'container_scanning_pipeline'
|
||||
time_frame:
|
||||
- 28d
|
||||
- all
|
||||
```
|
||||
See the [implement Service Ping](implement.md) guide.
|
||||
|
||||
## Example Service Ping payload
|
||||
|
||||
|
|
|
|||
|
|
@ -217,7 +217,7 @@ create ee/config/metrics/counts_7d/issues.yml
|
|||
|
||||
## Metrics added dynamic to Service Ping payload
|
||||
|
||||
The [Redis HLL metrics](index.md#known-events-are-added-automatically-in-service-data-payload) are added automatically to Service Ping payload.
|
||||
The [Redis HLL metrics](implement.md#known-events-are-added-automatically-in-service-data-payload) are added automatically to Service Ping payload.
|
||||
|
||||
A YAML metric definition is required for each metric. A dedicated generator is provided to create metric definitions for Redis HLL events.
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ The following guidelines explain the steps to follow at each stage of a metric's
|
|||
|
||||
## Add a new metric
|
||||
|
||||
Please follow the [Implementing Service Ping](index.md#implementing-service-ping) guide.
|
||||
Follow the [Implement Service Ping](implement.md) guide.
|
||||
|
||||
## Change an existing metric
|
||||
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ are regular backend changes.
|
|||
metrics that are based on Database.
|
||||
- For tracking using Redis HLL (HyperLogLog):
|
||||
- Check the Redis slot.
|
||||
- Check if a [feature flag is needed](index.md#recommendations).
|
||||
- Check if a [feature flag is needed](implement.md#recommendations).
|
||||
- For a metric's YAML definition:
|
||||
- Check the metric's `description`.
|
||||
- Check the metric's `key_path`.
|
||||
|
|
|
|||
|
|
@ -1138,7 +1138,8 @@ There are a few possible downsides to this:
|
|||
the last project that gets backed up.
|
||||
- Fork networks should be entirely read-only while the projects inside get backed up to prevent potential changes to the pool repository.
|
||||
|
||||
There is an **experimental** script that attempts to automate this process in [Snippet 2149205](https://gitlab.com/-/snippets/2149205).
|
||||
There is an **experimental** script that attempts to automate this process in
|
||||
[the Geo team Runbooks project](https://gitlab.com/gitlab-org/geo-team/runbooks/-/tree/main/experimental-online-backup-through-rsync).
|
||||
|
||||
## Backup and restore for installations using PgBouncer
|
||||
|
||||
|
|
|
|||
|
|
@ -358,6 +358,36 @@ NOTE:
|
|||
Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/)
|
||||
and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches.
|
||||
|
||||
### 14.2.0
|
||||
|
||||
- Due to an issue where `BatchedBackgroundMigrationWorkers` were
|
||||
[not working](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2785#note_614738345)
|
||||
for self-managed instances, a [fix was created](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65106)
|
||||
and a [14.0.Z](#1400) version was released. If you haven't updated to 14.0.Z, you need
|
||||
to update to at least 14.1.0 that contains the same fix before you update to
|
||||
to 14.2.
|
||||
- GitLab 14.2.0 contains background migrations to [address Primary Key overflow risk for tables with an integer PK](https://gitlab.com/groups/gitlab-org/-/epics/4785) for the tables listed below:
|
||||
|
||||
- [`ci_build_needs`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65216)
|
||||
- [`ci_build_trace_chunks`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66123)
|
||||
- [`ci_builds_runner_session`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66433)
|
||||
- [`deployments`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67341)
|
||||
- [`geo_job_artifact_deleted_events`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66763)
|
||||
- [`push_event_payloads`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67299)
|
||||
- `ci_job_artifacts`:
|
||||
- [Finalize job_id conversion to `bigint` for `ci_job_artifacts`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67774)
|
||||
- [Finalize `ci_job_artifacts` conversion to `bigint`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65601)
|
||||
|
||||
If the migrations are executed as part of a no-downtime deployment, there's a risk of failure due to lock conflicts with the application logic, resulting in lock timeout or deadlocks. In each case, these migrations are safe to re-run until successful:
|
||||
|
||||
```shell
|
||||
# For Omnibus GitLab
|
||||
sudo gitlab-rake db:migrate
|
||||
|
||||
# For source installations
|
||||
sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production
|
||||
```
|
||||
|
||||
### 14.1.0
|
||||
|
||||
- Due to an issue where `BatchedBackgroundMigrationWorkers` were
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ module API
|
|||
options[:project] = user_project
|
||||
|
||||
if Feature.enabled?(:api_caching_merge_requests, user_project, type: :development, default_enabled: :yaml)
|
||||
present_cached merge_requests, expires_in: 10.minutes, **options
|
||||
present_cached merge_requests, expires_in: 12.hours, **options
|
||||
else
|
||||
present merge_requests, options
|
||||
end
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@
|
|||
- name: design_action
|
||||
category: source_code
|
||||
aggregation: daily
|
||||
feature_flag: usage_data_design_action
|
||||
- name: project_action
|
||||
category: source_code
|
||||
aggregation: daily
|
||||
|
|
|
|||
|
|
@ -390,28 +390,13 @@ describe('Design management design index page', () => {
|
|||
);
|
||||
});
|
||||
|
||||
describe('with usage_data_design_action enabled', () => {
|
||||
it('tracks design view service ping', () => {
|
||||
createComponent(
|
||||
{ loading: true },
|
||||
{
|
||||
provide: {
|
||||
glFeatures: { usageDataDesignAction: true },
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(Api.trackRedisHllUserEvent).toHaveBeenCalledTimes(1);
|
||||
expect(Api.trackRedisHllUserEvent).toHaveBeenCalledWith(
|
||||
DESIGN_SERVICE_PING_EVENT_TYPES.DESIGN_ACTION,
|
||||
);
|
||||
});
|
||||
});
|
||||
it('tracks design view service ping', () => {
|
||||
createComponent({ loading: true });
|
||||
|
||||
describe('with usage_data_design_action disabled', () => {
|
||||
it("doesn't track design view service ping", () => {
|
||||
createComponent({ loading: true });
|
||||
expect(Api.trackRedisHllUserEvent).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
expect(Api.trackRedisHllUserEvent).toHaveBeenCalledTimes(1);
|
||||
expect(Api.trackRedisHllUserEvent).toHaveBeenCalledWith(
|
||||
DESIGN_SERVICE_PING_EVENT_TYPES.DESIGN_ACTION,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ RSpec.configure do |config|
|
|||
example.run
|
||||
|
||||
delete_from_all_tables!(except: deletion_except_tables)
|
||||
|
||||
self.class.use_transactional_tests = true
|
||||
end
|
||||
|
||||
config.around(:each, :migration) do |example|
|
||||
|
|
@ -60,5 +62,7 @@ RSpec.configure do |config|
|
|||
example.run
|
||||
|
||||
delete_from_all_tables!
|
||||
|
||||
self.class.use_transactional_tests = true
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue