kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2020-07-03 00:09:23 +00:00
parent 2c0e92d031
commit 57b359e90c
19 changed files with 426 additions and 784 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
app/controllers/concerns/metrics/dashboard/prometheus_api_proxy.rb Normal file
View File

@ -0,0 +1,53 @@
# frozen_string_literal: true
module Metrics::Dashboard::PrometheusApiProxy
extend ActiveSupport::Concern
include RenderServiceResults
included do
before_action :authorize_read_prometheus!, only: [:prometheus_proxy]
end
def prometheus_proxy
variable_substitution_result =
proxy_variable_substitution_service.new(proxyable, permit_params).execute
if variable_substitution_result[:status] == :error
return error_response(variable_substitution_result)
end
prometheus_result = Prometheus::ProxyService.new(
proxyable,
proxy_method,
proxy_path,
variable_substitution_result[:params]
).execute
return continue_polling_response if prometheus_result.nil?
return error_response(prometheus_result) if prometheus_result[:status] == :error
success_response(prometheus_result)
end
private
def proxyable
raise NotImplementedError, "#{self.class} must implement method: #{__callee__}"
end
def proxy_variable_substitution_service
raise NotImplementedError, "#{self.class} must implement method: #{__callee__}"
end
def permit_params
params.permit!
end
def proxy_method
request.method
end
def proxy_path
params[:proxy_path]
end
end

48
app/controllers/projects/environments/prometheus_api_controller.rb
View File

@ -1,51 +1,17 @@
# frozen_string_literal: true
class Projects::Environments::PrometheusApiController < Projects::ApplicationController
include RenderServiceResults
include Metrics::Dashboard::PrometheusApiProxy
before_action :authorize_read_prometheus!
before_action :environment
def proxy
variable_substitution_result =
variable_substitution_service.new(environment, permit_params).execute
if variable_substitution_result[:status] == :error
return error_response(variable_substitution_result)
end
prometheus_result = Prometheus::ProxyService.new(
environment,
proxy_method,
proxy_path,
variable_substitution_result[:params]
).execute
return continue_polling_response if prometheus_result.nil?
return error_response(prometheus_result) if prometheus_result[:status] == :error
success_response(prometheus_result)
end
before_action :proxyable
private
def variable_substitution_service
def proxyable
@proxyable ||= project.environments.find(params[:id])
end
def proxy_variable_substitution_service
Prometheus::ProxyVariableSubstitutionService
end
def permit_params
params.permit!
end
def environment
@environment ||= project.environments.find(params[:id])
end
def proxy_method
request.method
end
def proxy_path
params[:proxy_path]
end
end

2
app/models/audit_event.rb
View File

@ -4,7 +4,7 @@ class AuditEvent < ApplicationRecord
include CreatedAtFilterable
include IgnorableColumns
ignore_column :updated_at, remove_with: '13.3', remove_after: '2020-08-22'
ignore_column :updated_at, remove_with: '13.4', remove_after: '2020-09-22'
serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize

2
app/models/user.rb
View File

@ -69,8 +69,6 @@ class User < ApplicationRecord
MINIMUM_INACTIVE_DAYS = 180
ignore_column :ghost, remove_with: '13.2', remove_after: '2020-06-22'
# Override Devise::Models::Trackable#update_tracked_fields!
# to limit database writes to at most once every hour
# rubocop: disable CodeReuse/ServiceClass

291
bin/feature-flag
View File

@ -1,291 +0,0 @@
#!/usr/bin/env ruby
#
# Generate a feature flag entry file in the correct location.
#
# Automatically stages the file and amends the previous commit if the `--amend`
# argument is used.
require 'optparse'
require 'yaml'
require 'fileutils'
require 'cgi'
require_relative '../lib/feature/shared' unless defined?(Feature::Shared)
Options = Struct.new(
:name,
:type,
:group,
:ee,
:amend,
:dry_run,
:force,
:introduced_by_url,
:rollout_issue_url
)
module FeatureFlagHelpers
Abort = Class.new(StandardError)
Done = Class.new(StandardError)
def capture_stdout(cmd)
output = IO.popen(cmd, &:read)
fail_with "command failed: #{cmd.join(' ')}" unless $?.success?
output
end
def fail_with(message)
raise Abort, "\e[31merror\e[0m #{message}"
end
end
class FeatureFlagOptionParser
extend FeatureFlagHelpers
extend ::Feature::Shared
class << self
def parse(argv)
options = Options.new
parser = OptionParser.new do |opts|
opts.banner = "Usage: #{__FILE__} [options] <feature-flag>\n\n"
# Note: We do not provide a shorthand for this in order to match the `git
# commit` interface
opts.on('--amend', 'Amend the previous commit') do |value|
options.amend = value
end
opts.on('-f', '--force', 'Overwrite an existing entry') do |value|
options.force = value
end
opts.on('-m', '--introduced-by-url [string]', String, 'URL to Merge Request introducing Feature Flag') do |value|
options.introduced_by_url = value
end
opts.on('-i', '--rollout-issue-url [string]', String, 'URL to Issue rolling out Feature Flag') do |value|
options.rollout_issue_url = value
end
opts.on('-n', '--dry-run', "Don't actually write anything, just print") do |value|
options.dry_run = value
end
opts.on('-g', '--group [string]', String, "The group introducing a feature flag, like: `group::apm`") do |value|
options.group = value if value.start_with?('group::')
end
opts.on('-t', '--type [string]', String, "The category of the feature flag, valid options are: #{TYPES.keys.map(&:to_s).join(', ')}") do |value|
options.type = value.to_sym if TYPES[value.to_sym]
end
opts.on('-e', '--ee', 'Generate a feature flag entry for GitLab EE') do |value|
options.ee = value
end
opts.on('-h', '--help', 'Print help message') do
$stdout.puts opts
raise Done.new
end
end
parser.parse!(argv)
unless argv.one?
$stdout.puts parser.help
$stdout.puts
raise Abort, 'Feature flag name is required'
end
# Name is a first name
options.name = argv.first
options
end
def read_group
$stdout.puts ">> Please specify the group introducing feature flag, like `group::apm`:"
loop do
$stdout.print "\n?> "
group = $stdin.gets.strip
group = nil if group.empty?
return group if group.nil? || group.start_with?('group::')
$stderr.puts "Group needs to include `group::`"
end
end
def read_type
$stdout.puts ">> Please specify the type of your feature flag:"
$stdout.puts
TYPES.each do |type, data|
$stdout.puts "#{type.to_s.rjust(15)}#{' '*6}#{data[:description]}"
end
loop do
$stdout.print "\n?> "
type = $stdin.gets.strip.to_sym
return type if TYPES[type]
$stderr.puts "Invalid type specified '#{type}'"
end
end
def read_issue_url(options)
return unless TYPES.dig(options.type, :rollout_issue)
url = "https://gitlab.com/gitlab-org/gitlab/-/issues/new"
title = "[Feature flag] Rollout of `#{options.name}`"
description = File.read('.gitlab/issue_templates/Feature Flag Roll Out.md')
description.sub!(':feature_name', options.name)
issue_new_url = url + "?" +
"issue[title]=" + CGI.escape(title) + "&"
# TODO: We should be able to pick `issueable_template`
# + "issue[description]=" + CGI.escape(description)
$stdout.puts ">> Open this URL and fill the rest of details:"
$stdout.puts issue_new_url
$stdout.puts
$stdout.puts ">> Paste URL here, or enter to skip:"
loop do
$stdout.print "\n?> "
created_url = $stdin.gets.strip
created_url = nil if created_url.empty?
return created_url if created_url.nil? || created_url.start_with?('https://')
$stderr.puts "URL needs to start with https://"
end
end
end
end
class FeatureFlagCreator
include FeatureFlagHelpers
attr_reader :options
def initialize(options)
@options = options
end
def execute
assert_feature_branch!
assert_name!
assert_existing_feature_flag!
# Read type from $stdin unless is already set
options.type ||= FeatureFlagOptionParser.read_type
options.group ||= FeatureFlagOptionParser.read_group
options.rollout_issue_url ||= FeatureFlagOptionParser.read_issue_url(options)
$stdout.puts "\e[32mcreate\e[0m #{file_path}"
$stdout.puts contents
unless options.dry_run
write
amend_commit if options.amend
end
if editor
system("#{editor} '#{file_path}'")
end
end
private
def contents
YAML.dump(
'name' => options.name,
'introduced_by_url' => options.introduced_by_url,
'rollout_issue_url' => options.rollout_issue_url,
'group' => options.group.to_s,
'type' => options.type.to_s,
'default_enabled' => false
).strip
end
def write
FileUtils.mkdir_p(File.dirname(file_path))
File.write(file_path, contents)
end
def editor
ENV['EDITOR']
end
def amend_commit
fail_with "git add failed" unless system(*%W[git add #{file_path}])
Kernel.exec(*%w[git commit --amend])
end
def assert_feature_branch!
return unless branch_name == 'master'
fail_with "Create a branch first!"
end
def assert_existing_feature_flag!
existing_path = all_feature_flag_names[options.name]
return unless existing_path
return if options.force
fail_with "#{existing_path} already exists! Use `--force` to overwrite."
end
def assert_name!
return if options.name.match(/\A[a-z0-9_-]+\Z/)
fail_with "Provide a name for the feature flag that is [a-z0-9_-]"
end
def file_path
feature_flags_paths.last
.sub('**', options.type.to_s)
.sub('*.yml', options.name + '.yml')
end
def all_feature_flag_names
@all_feature_flag_names ||=
feature_flags_paths.map do |glob_path|
Dir.glob(glob_path).map do |path|
[File.basename(path, '.yml'), path]
end
end.flatten(1).to_h
end
def feature_flags_paths
paths = []
paths << File.join('config', 'feature_flags', '**', '*.yml')
paths << File.join('ee', 'config', 'feature_flags', '**', '*.yml') if ee?
paths
end
def ee?
options.ee
end
def branch_name
@branch_name ||= capture_stdout(%w[git symbolic-ref --short HEAD]).strip
end
end
if $0 == __FILE__
begin
options = FeatureFlagOptionParser.parse(ARGV)
FeatureFlagCreator.new(options).execute
rescue FeatureFlagHelpers::Abort => ex
$stderr.puts ex.message
exit 1
rescue FeatureFlagHelpers::Done
exit
end
end
# vim: ft=ruby

5
changelogs/unreleased/update-auto-build-image-to-0-3-0.yml Normal file
View File

@ -0,0 +1,5 @@
---
title: Add custom Dockerfile paths to Auto DevOps Build stage with DOCKERFILE_PATH
merge_request: 35662
author: thklein
type: added

2
config/routes/project.rb
View File

@ -257,7 +257,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
# This route is also defined in gitlab-workhorse. Make sure to update accordingly.
get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false
get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api
get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#prometheus_proxy', as: :prometheus_api
get '/sample_metrics', to: 'environments/sample_metrics#query'
end

8
danger/changelog/Dangerfile
View File

@ -35,7 +35,11 @@ def check_changelog_yaml(path)
fail "`title` should be set, in #{gitlab.html_link(path)}! #{SEE_DOC}" if yaml["title"].nil?
fail "`type` should be set, in #{gitlab.html_link(path)}! #{SEE_DOC}" if yaml["type"].nil?
if yaml["merge_request"].nil? && !helper.security_mr?
return if helper.security_mr?
cherry_pick_against_stable_branch = helper.cherry_pick_mr? && helper.stable_branch?
if yaml["merge_request"].nil?
mr_line = raw_file.lines.find_index("merge_request:\n")
if mr_line
@ -43,7 +47,7 @@ def check_changelog_yaml(path)
else
message "Consider setting `merge_request` to #{gitlab.mr_json["iid"]} in #{gitlab.html_link(path)}. #{SEE_DOC}"
end
elsif yaml["merge_request"] != gitlab.mr_json["iid"] && !helper.security_mr?
elsif yaml["merge_request"] != gitlab.mr_json["iid"] && !cherry_pick_against_stable_branch
fail "Merge request ID was not set to #{gitlab.mr_json["iid"]}! #{SEE_DOC}"
end
rescue Psych::SyntaxError, Psych::DisallowedClass, Psych::BadAlias

107
doc/administration/monitoring/performance/grafana_configuration.md
View File

@ -6,85 +6,90 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Grafana Configuration
[Grafana](https://grafana.com/) is a tool that allows you to visualize time
series metrics through graphs and dashboards. GitLab writes performance data to Prometheus
and Grafana will allow you to query to display useful graphs.
[Grafana](https://grafana.com/) is a tool that enables you to visualize time
series metrics through graphs and dashboards. GitLab writes performance data to Prometheus,
and Grafana allows you to query the data to display useful graphs.
## Installation
[Omnibus GitLab can help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html)
Omnibus GitLab can [help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html)
or Grafana supplies package repositories (Yum/Apt) for easy installation.
See [Grafana installation documentation](https://grafana.com/docs/grafana/latest/installation/)
for detailed steps.
NOTE: **Note:**
Before starting Grafana for the first time, set the admin user
and password in `/etc/grafana/grafana.ini`. Otherwise, the default password
will be `admin`.
and password in `/etc/grafana/grafana.ini`. If you don't, the default password
is `admin`.
## Configuration
Login as the admin user. Expand the menu by clicking the Grafana logo in the
top left corner. Choose 'Data Sources' from the menu. Then, click 'Add new'
in the top bar.
![Grafana empty data source page](img/grafana_data_source_empty.png)
![Grafana data source configurations](img/grafana_data_source_configuration.png)
1. Log in to Grafana as the admin user.
1. Expand the menu by clicking the Grafana logo in the top left corner.
1. Choose **Data Sources** from the menu.
1. Click **Add new** in the top bar:
![Grafana empty data source page](img/grafana_data_source_empty.png)
1. Edit the data source to fit your needs:
![Grafana data source configurations](img/grafana_data_source_configuration.png)
1. Click **Save**.
## Import Dashboards
You can now import a set of default dashboards that will give you a good
start on displaying useful information. GitLab has published a set of default
[Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) to get you started. Clone the
repository or download a zip/tarball, then follow these steps to import each
JSON file.
You can now import a set of default dashboards to start displaying useful information.
GitLab has published a set of default
[Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) to get you started.
Clone the repository, or download a ZIP file or tarball, then follow these steps to import each
JSON file individually:
Open the dashboard dropdown menu and click 'Import'
1. Log in to Grafana as the admin user.
1. Open the dashboard dropdown menu and click **Import**:
![Grafana dashboard dropdown](img/grafana_dashboard_dropdown.png)
1. Click **Choose file**, and browse to the location where you downloaded or
cloned the dashboard repository. Select a JSON file to import:
![Grafana dashboard import](img/grafana_dashboard_import.png)
1. After the dashboard is imported, click the **Save dashboard** icon in the top bar:
![Grafana save icon](img/grafana_save_icon.png)
![Grafana dashboard dropdown](img/grafana_dashboard_dropdown.png)
Click 'Choose file' and browse to the location where you downloaded or cloned
the dashboard repository. Pick one of the JSON files to import.
![Grafana dashboard import](img/grafana_dashboard_import.png)
Once the dashboard is imported, be sure to click save icon in the top bar. If
you do not save the dashboard after importing it will be removed when you
navigate away.
![Grafana save icon](img/grafana_save_icon.png)
NOTE: **Note:**
If you don't save the dashboard after importing it, the dashboard is removed
when you navigate away from the page.
Repeat this process for each dashboard you wish to import.
Alternatively you can automatically import all the dashboards into your Grafana
instance. See the README of the [Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards)
repository for more information on this process.
Alternatively, you can import all the dashboards into your Grafana
instance. For more information about this process, see the
[README of the Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards)
repository.
## Integration with GitLab UI
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61005) in GitLab 12.1.
If you have set up Grafana, you can enable a link to access it easily from the sidebar:
After setting up Grafana, you can enable a link to access it easily from the
GitLab sidebar:
1. Go to the **Admin Area > Settings > Metrics and profiling**.
1. Navigate to the **{admin}** **Admin Area > Settings > Metrics and profiling**.
1. Expand **Metrics - Grafana**.
1. Check the "Enable access to Grafana" checkbox.
1. If Grafana is enabled through Omnibus GitLab and on the same server,
leave **Grafana URL** unchanged. It should be `/-/grafana`.
In any other case, enter the full URL of the Grafana instance.
1. Check the **Enable access to Grafana** checkbox.
1. Configure the **Grafana URL**:
- *If Grafana is enabled through Omnibus GitLab and on the same server,*
leave **Grafana URL** unchanged. It should be `/-/grafana`.
- *Otherwise,* enter the full URL of the Grafana instance.
1. Click **Save changes**.
1. The new link will be available in the **Admin Area > Monitoring > Metrics Dashboard**.
GitLab displays your link in the **{admin}** **Admin Area > Monitoring > Metrics Dashboard**.
## Security Update
Users running GitLab version 12.0 or later should immediately upgrade to one of the following security releases due to a known vulnerability with the embedded Grafana dashboard:
Users running GitLab version 12.0 or later should immediately upgrade to one of the
following security releases due to a known vulnerability with the embedded Grafana dashboard:
- 12.0.6
- 12.1.6
After upgrading, the Grafana dashboard will be disabled and the location of your existing Grafana data will be changed from `/var/opt/gitlab/grafana/data/` to `/var/opt/gitlab/grafana/data.bak.#{Date.today}/`.
After upgrading, the Grafana dashboard is disabled, and the location of your
existing Grafana data is changed from `/var/opt/gitlab/grafana/data/` to
`/var/opt/gitlab/grafana/data.bak.#{Date.today}/`.
To prevent the data from being relocated, you can run the following command prior to upgrading:
@ -100,19 +105,23 @@ sudo mv /var/opt/gitlab/grafana/data.bak.xxxx/ /var/opt/gitlab/grafana/data/
However, you should **not** reinstate your old data _except_ under one of the following conditions:
1. If you are certain that you changed your default admin password when you enabled Grafana
1. If you run GitLab in a private network, accessed only by trusted users, and your Grafana login page has not been exposed to the internet
1. If you're certain that you changed your default admin password when you enabled Grafana.
1. If you run GitLab in a private network, accessed only by trusted users, and your
Grafana login page has not been exposed to the internet.
If you require access to your old Grafana data but do not meet one of these criteria, you may consider:
If you require access to your old Grafana data but don't meet one of these criteria, you may consider:
1. Reinstating it temporarily.
1. [Exporting the dashboards](https://grafana.com/docs/grafana/latest/reference/export_import/#exporting-a-dashboard) you need.
1. Refreshing the data and [re-importing your dashboards](https://grafana.com/docs/grafana/latest/reference/export_import/#importing-a-dashboard).
DANGER: **Danger:**
This poses a temporary vulnerability while your old Grafana data is in use and the decision to do so should be weighed carefully with your need to access existing data and dashboards.
These actions pose a temporary vulnerability while your old Grafana data is in use.
Deciding to take any of these actions should be weighed carefully with your need to access
existing data and dashboards.
For more information and further mitigation details, please refer to our [blog post on the security release](https://about.gitlab.com/releases/2019/08/12/critical-security-release-gitlab-12-dot-1-dot-6-released/).
For more information and further mitigation details, please refer to our
[blog post on the security release](https://about.gitlab.com/releases/2019/08/12/critical-security-release-gitlab-12-dot-1-dot-6-released/).
---

20
doc/administration/monitoring/prometheus/gitlab_exporter.md
View File

@ -9,27 +9,25 @@ info: To determine the technical writer assigned to the Stage/Group associated w
>- Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1132).
>- Renamed from `GitLab monitor exporter` to `GitLab exporter` in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16511).
The [GitLab exporter](https://gitlab.com/gitlab-org/gitlab-exporter) allows you to
measure various GitLab metrics, pulled from Redis and the database, in Omnibus GitLab
The [GitLab exporter](https://gitlab.com/gitlab-org/gitlab-exporter) enables you to
measure various GitLab metrics pulled from Redis and the database in Omnibus GitLab
instances.
NOTE: **Note:**
For installations from source you'll have to install and configure it yourself.
For installations from source you must install and configure it yourself.
To enable the GitLab exporter in an Omnibus GitLab instance:
1. [Enable Prometheus](index.md#configuring-prometheus)
1. Edit `/etc/gitlab/gitlab.rb`
1. Add or find and uncomment the following line, making sure it's set to `true`:
1. [Enable Prometheus](index.md#configuring-prometheus).
1. Edit `/etc/gitlab/gitlab.rb`.
1. Add, or find and uncomment, the following line, making sure it's set to `true`:
```ruby
gitlab_exporter['enable'] = true
```
1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure)
for the changes to take effect
for the changes to take effect.
Prometheus will now automatically begin collecting performance data from
the GitLab exporter exposed under `localhost:9168`.
[← Back to the main Prometheus page](index.md)
Prometheus automatically begins collecting performance data from
the GitLab exporter exposed at `localhost:9168`.

38
doc/administration/monitoring/prometheus/gitlab_metrics.md
View File

@ -14,18 +14,18 @@ To enable the GitLab Prometheus metrics:
1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect.
NOTE: **Note:**
For installations from source you'll have to configure it yourself.
For installations from source you must configure it yourself.
## Collecting the metrics
GitLab monitors its own internal service metrics, and makes them available at the
`/-/metrics` endpoint. Unlike other [Prometheus](https://prometheus.io) exporters, to access
it, the client IP address needs to be [explicitly allowed](../ip_whitelist.md).
the metrics, the client IP address must be [explicitly allowed](../ip_whitelist.md).
For [Omnibus GitLab](https://docs.gitlab.com/omnibus/) and Chart installations,
these metrics are enabled and collected as of
[GitLab 9.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1702).
For source installations or earlier versions, these metrics must be enabled
For source installations, these metrics must be enabled
manually and collected by a Prometheus server.
For enabling and viewing metrics from Sidekiq nodes, see [Sidekiq metrics](#sidekiq-metrics).
@ -40,7 +40,7 @@ The following metrics are available:
| `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output does not exist | `controller`, `action` |
| `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` |
| `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | |
| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | `controller`, `action`, `operation` |
| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation` |
| `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | |
| `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` |
| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` |
@ -92,9 +92,9 @@ The following metrics are available:
| `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` |
| `http_requests_total` | Counter | 9.4 | Rack request count | `method` |
| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method`, `status` |
| `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of sql calls | `controller`, `action` |
| `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write sql calls | `controller`, `action` |
| `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached sql calls | `controller`, `action` |
| `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of SQL calls | `controller`, `action` |
| `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write SQL calls | `controller`, `action` |
| `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached SQL calls | `controller`, `action` |
| `http_redis_requests_duration_seconds` | Histogram | 13.1 | Redis requests duration during web transactions | `controller`, `action` |
| `http_redis_requests_total` | Counter | 13.1 | Redis requests count during web transactions | `controller`, `action` |
| `http_elasticsearch_requests_duration_seconds` **(STARTER)** | Histogram | 13.1 | Elasticsearch requests duration during web transactions | `controller`, `action` |
@ -120,7 +120,7 @@ The following metrics can be controlled by feature flags:
## Sidekiq metrics
Sidekiq jobs may also gather metrics, and these metrics can be accessed if the
Sidekiq exporter is enabled (for example, using the `monitoring.sidekiq_exporter`
Sidekiq exporter is enabled: for example, using the `monitoring.sidekiq_exporter`
configuration option in `gitlab.yml`. These metrics are served from the
`/metrics` path on the configured port.
@ -187,16 +187,16 @@ The following metrics are available:
## Connection pool metrics
These metrics record the status of the database [connection pools](https://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html).
These metrics record the status of the database
[connection pools](https://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html),
and the metrics all have these labels:
They all have these labels:
1. `class` - the Ruby class being recorded.
1. `ActiveRecord::Base` is the main database connection.
1. `Geo::TrackingBase` is the connection to the Geo tracking database, if
enabled.
1. `host` - the host name used to connect to the database.
1. `port` - the port used to connect to the database.
- `class` - the Ruby class being recorded.
- `ActiveRecord::Base` is the main database connection.
- `Geo::TrackingBase` is the connection to the Geo tracking database, if
enabled.
- `host` - the host name used to connect to the database.
- `port` - the port used to connect to the database.
| Metric | Type | Since | Description |
|:----------------------------------------------|:------|:------|:--------------------------------------------------|
@ -256,10 +256,10 @@ When Puma is used instead of Unicorn, the following metrics are available:
GitLab's Prometheus client requires a directory to store metrics data shared between multi-process services.
Those files are shared among all instances running under Unicorn server.
The directory must be accessible to all running Unicorn's processes, or
metrics won't function correctly.
metrics can't function correctly.
This directory's location is configured using environment variable `prometheus_multiproc_dir`.
For best performance, create this directory in `tmpfs`.
If GitLab is installed using [Omnibus GitLab](https://docs.gitlab.com/omnibus/)
and `tmpfs` is available, then the metrics directory will be configured for you.
and `tmpfs` is available, then GitLab configures the metrics directory for you.

6
doc/topics/autodevops/customize.md
View File

@ -41,11 +41,16 @@ If your goal is to use only a single custom buildpack, you should provide the pr
## Custom `Dockerfile`
> Support for `DOCKERFILE_PATH` was [added in GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35662)
If your project has a `Dockerfile` in the root of the project repository, Auto DevOps
builds a Docker image based on the Dockerfile, rather than using buildpacks.
This can be much faster and result in smaller images, especially if your
Dockerfile is based on [Alpine](https://hub.docker.com/_/alpine/).
If you set the `DOCKERFILE_PATH` CI variable, Auto Build looks for a Dockerfile there
instead.
## Passing arguments to `docker build`
Arguments can be passed to the `docker build` command using the
@ -311,6 +316,7 @@ applications.
| `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). |
| `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. |
| `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. |
| `DOCKERFILE_PATH` | From GitLab 13.2, allows overriding the [default Dockerfile path for the build stage](#custom-dockerfile) |
| `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. |
| `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. |
| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes won't prevent word splitting. |

5
doc/user/clusters/applications.md
View File

@ -667,6 +667,11 @@ To install applications using GitLab CI/CD:
- template: Managed-Cluster-Applications.gitlab-ci.yml
```
NOTE: **Note:**
The job provided by this template connects to the cluster using tools provided
in a custom Docker image. It requires that you have a runner registered with the Docker,
Kubernetes, or Docker Machine executor.
1. Add a `.gitlab/managed-apps/config.yaml` file to define which
applications you would like to install. Define the `installed` key as
`true` to install the application and `false` to uninstall the

2
lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
View File

@ -1,6 +1,6 @@
build:
stage: build
image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.3"
image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.3.0"
variables:
DOCKER_TLS_CERTDIR: ""
services:

12
lib/gitlab/danger/helper.rb
View File

@ -196,6 +196,18 @@ module Gitlab
gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/')
end
def cherry_pick_mr?
return false unless gitlab_helper
/cherry[\s-]*pick/i.match?(gitlab_helper.mr_json['title'])
end
def stable_branch?
return false unless gitlab_helper
/\A\d+-\d+-stable-ee/i.match?(gitlab_helper.mr_json['target_branch'])
end
def mr_has_labels?(*labels)
return false unless gitlab_helper

191
spec/bin/feature_flag_spec.rb
View File

@ -1,191 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
load File.expand_path('../../bin/feature-flag', __dir__)
RSpec.describe 'bin/feature-flag' do
using RSpec::Parameterized::TableSyntax
describe FeatureFlagCreator do
let(:argv) { %w[feature-flag-name -t development -g group::memory -i https://url] }
let(:options) { FeatureFlagOptionParser.parse(argv) }
let(:creator) { described_class.new(options) }
let(:existing_flag) { File.join('config', 'feature_flags', 'development', 'existing-feature-flag.yml') }
before do
# create a dummy feature flag
FileUtils.mkdir_p(File.dirname(existing_flag))
File.write(existing_flag, '{}')
# ignore writes
allow(File).to receive(:write).and_return(true)
# ignore stdin
allow($stdin).to receive(:gets).and_raise('EOF')
# ignore Git commands
allow(creator).to receive(:branch_name) { 'feature-branch' }
end
after do
FileUtils.rm_f(existing_flag)
end
subject { creator.execute }
it 'properly creates a feature flag' do
expect(File).to receive(:write).with(
File.join('config', 'feature_flags', 'development', 'feature-flag-name.yml'),
anything)
expect do
subject
end.to output(/name: feature-flag-name/).to_stdout
end
context 'when running on master' do
it 'requires feature branch' do
expect(creator).to receive(:branch_name) { 'master' }
expect { subject }.to raise_error(FeatureFlagHelpers::Abort, /Create a branch first/)
end
end
context 'validates feature flag name' do
where(:argv, :ex) do
%w[.invalid.feature.flag] | /Provide a name for the feature flag that is/
%w[existing-feature-flag] | /already exists!/
end
with_them do
it do
expect { subject }.to raise_error(ex)
end
end
end
end
describe FeatureFlagOptionParser do
describe '.parse' do
where(:param, :argv, :result) do
:name | %w[foo] | 'foo'
:amend | %w[foo --amend] | true
:force | %w[foo -f] | true
:force | %w[foo --force] | true
:ee | %w[foo -e] | true
:ee | %w[foo --ee] | true
:introduced_by_url | %w[foo -m https://url] | 'https://url'
:introduced_by_url | %w[foo --introduced-by-url https://url] | 'https://url'
:rollout_issue_url | %w[foo -i https://url] | 'https://url'
:rollout_issue_url | %w[foo --rollout-issue-url https://url] | 'https://url'
:dry_run | %w[foo -n] | true
:dry_run | %w[foo --dry-run] | true
:type | %w[foo -t development] | :development
:type | %w[foo --type development] | :development
:type | %w[foo -t invalid] | nil
:type | %w[foo --type invalid] | nil
:group | %w[foo -g group::memory] | 'group::memory'
:group | %w[foo --group group::memory] | 'group::memory'
:group | %w[foo -g invalid] | nil
:group | %w[foo --group invalid] | nil
end
with_them do
it do
options = described_class.parse(Array(argv))
expect(options.public_send(param)).to eq(result)
end
end
it 'missing feature flag name' do
expect do
expect { described_class.parse(%w[--amend]) }.to output(/Feature flag name is required/).to_stdout
end.to raise_error(FeatureFlagHelpers::Abort)
end
it 'parses -h' do
expect do
expect { described_class.parse(%w[foo -h]) }.to output(/Usage:/).to_stdout
end.to raise_error(FeatureFlagHelpers::Done)
end
end
describe '.read_type' do
let(:type) { 'development' }
it 'reads type from $stdin' do
expect($stdin).to receive(:gets).and_return(type)
expect do
expect(described_class.read_type).to eq(:development)
end.to output(/specify the type/).to_stdout
end
context 'invalid type given' do
let(:type) { 'invalid' }
it 'shows error message and retries' do
expect($stdin).to receive(:gets).and_return(type)
expect($stdin).to receive(:gets).and_raise('EOF')
expect do
expect { described_class.read_type }.to raise_error(/EOF/)
end.to output(/specify the type/).to_stdout
.and output(/Invalid type specified/).to_stderr
end
end
end
describe '.read_group' do
let(:group) { 'group::memory' }
it 'reads type from $stdin' do
expect($stdin).to receive(:gets).and_return(group)
expect do
expect(described_class.read_group).to eq('group::memory')
end.to output(/specify the group/).to_stdout
end
context 'invalid group given' do
let(:type) { 'invalid' }
it 'shows error message and retries' do
expect($stdin).to receive(:gets).and_return(type)
expect($stdin).to receive(:gets).and_raise('EOF')
expect do
expect { described_class.read_group }.to raise_error(/EOF/)
end.to output(/specify the group/).to_stdout
.and output(/Group needs to include/).to_stderr
end
end
end
describe '.rollout_issue_url' do
let(:options) { OpenStruct.new(name: 'foo', type: :development) }
let(:url) { 'https://issue' }
it 'reads type from $stdin' do
expect($stdin).to receive(:gets).and_return(url)
expect do
expect(described_class.read_issue_url(options)).to eq('https://issue')
end.to output(/Paste URL here/).to_stdout
end
context 'invalid URL given' do
let(:type) { 'invalid' }
it 'shows error message and retries' do
expect($stdin).to receive(:gets).and_return(type)
expect($stdin).to receive(:gets).and_raise('EOF')
expect do
expect { described_class.read_issue_url(options) }.to raise_error(/EOF/)
end.to output(/Paste URL here/).to_stdout
.and output(/URL needs to start/).to_stderr
end
end
end
end
end

208
spec/controllers/projects/environments/prometheus_api_controller_spec.rb
View File

@ -3,215 +3,73 @@
require 'spec_helper'
RSpec.describe Projects::Environments::PrometheusApiController do
let_it_be(:project) { create(:project) }
let_it_be(:environment) { create(:environment, project: project) }
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project) }
let_it_be(:proxyable) { create(:environment, project: project) }
before do
project.add_reporter(user)
sign_in(user)
end
describe 'GET #proxy' do
let(:prometheus_proxy_service) { instance_double(Prometheus::ProxyService) }
let(:expected_params) do
ActionController::Parameters.new(
environment_params(
proxy_path: 'query',
controller: 'projects/environments/prometheus_api',
action: 'proxy'
)
).permit!
end
context 'with valid requests' do
before do
allow(Prometheus::ProxyService).to receive(:new)
.with(environment, 'GET', 'query', expected_params)
.and_return(prometheus_proxy_service)
allow(prometheus_proxy_service).to receive(:execute)
.and_return(service_result)
describe 'GET #prometheus_proxy' do
it_behaves_like 'metrics dashboard prometheus api proxy' do
let(:proxyable_params) do
{
id: proxyable.id.to_s,
namespace_id: project.namespace.full_path,
project_id: project.name
}
end
context 'with success result' do
let(:service_result) { { status: :success, body: prometheus_body } }
context 'with variables' do
let(:prometheus_body) { '{"status":"success"}' }
let(:prometheus_json_body) { Gitlab::Json.parse(prometheus_body) }
let(:pod_name) { "pod1" }
it 'returns prometheus response' do
get :proxy, params: environment_params
before do
expected_params[:query] = %{up{pod_name="#{pod_name}"}}
expected_params[:variables] = { 'pod_name' => pod_name }
end
it 'replaces variables with values' do
get :prometheus_proxy, params: prometheus_proxy_params.merge(
query: 'up{pod_name="{{pod_name}}"}', variables: { 'pod_name' => pod_name }
)
expect(response).to have_gitlab_http_status(:success)
expect(Prometheus::ProxyService).to have_received(:new)
.with(environment, 'GET', 'query', expected_params)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to eq(prometheus_json_body)
.with(proxyable, 'GET', 'query', expected_params)
end
context 'with format string' do
before do
expected_params[:query] = %{up{environment="#{environment.slug}"}}
end
it 'replaces variables with values' do
get :proxy, params: environment_params.merge(query: 'up{environment="{{ci_environment_slug}}"}')
expect(Prometheus::ProxyService).to have_received(:new)
.with(environment, 'GET', 'query', expected_params)
end
context 'with nil query' do
let(:params_without_query) do
environment_params.except(:query)
end
before do
expected_params.delete(:query)
end
it 'does not raise error' do
get :proxy, params: params_without_query
expect(Prometheus::ProxyService).to have_received(:new)
.with(environment, 'GET', 'query', expected_params)
end
end
end
context 'with variables' do
let(:pod_name) { "pod1" }
before do
expected_params[:query] = %{up{pod_name="#{pod_name}"}}
expected_params[:variables] = { 'pod_name' => pod_name }
end
it 'replaces variables with values' do
get :proxy, params: environment_params.merge(
query: 'up{pod_name="{{pod_name}}"}', variables: { 'pod_name' => pod_name }
context 'with invalid variables' do
let(:params_with_invalid_variables) do
prometheus_proxy_params.merge(
query: 'up{pod_name="{{pod_name}}"}', variables: ['a']
)
expect(response).to have_gitlab_http_status(:success)
expect(Prometheus::ProxyService).to have_received(:new)
.with(environment, 'GET', 'query', expected_params)
end
context 'with invalid variables' do
let(:params_with_invalid_variables) do
environment_params.merge(
query: 'up{pod_name="{{pod_name}}"}', variables: ['a']
)
end
it 'returns 400' do
get :proxy, params: params_with_invalid_variables
expect(response).to have_gitlab_http_status(:bad_request)
expect(Prometheus::ProxyService).not_to receive(:new)
end
end
end
end
context 'with nil result' do
let(:service_result) { nil }
it 'returns 204 no_content' do
get :proxy, params: environment_params
expect(json_response['status']).to eq(_('processing'))
expect(json_response['message']).to eq(_('Not ready yet. Try again later.'))
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'with 404 result' do
let(:service_result) { { http_status: 404, status: :success, body: '{"body": "value"}' } }
it 'returns body' do
get :proxy, params: environment_params
expect(response).to have_gitlab_http_status(:not_found)
expect(json_response['body']).to eq('value')
end
end
context 'with error result' do
context 'with http_status' do
let(:service_result) do
{ http_status: :service_unavailable, status: :error, message: 'error message' }
end
it 'sets the http response status code' do
get :proxy, params: environment_params
expect(response).to have_gitlab_http_status(:service_unavailable)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
end
end
context 'without http_status' do
let(:service_result) { { status: :error, message: 'error message' } }
it 'returns bad_request' do
get :proxy, params: environment_params
it 'returns 400' do
get :prometheus_proxy, params: params_with_invalid_variables
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
expect(Prometheus::ProxyService).not_to receive(:new)
end
end
end
end
context 'with inappropriate requests' do
context 'with anonymous user' do
let(:prometheus_body) { nil }
before do
sign_out(user)
end
it 'redirects to signin page' do
get :proxy, params: environment_params
get :prometheus_proxy, params: prometheus_proxy_params
expect(response).to redirect_to(new_user_session_path)
end
end
context 'without correct permissions' do
before do
project.team.truncate
end
it 'returns 404' do
get :proxy, params: environment_params
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
context 'with invalid environment id' do
let(:other_environment) { create(:environment) }
it 'returns 404' do
get :proxy, params: environment_params(id: other_environment.id)
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
private
def environment_params(params = {})
{
id: environment.id.to_s,
namespace_id: project.namespace.full_path,
project_id: project.name,
proxy_path: 'query',
query: '1'
}.merge(params)
end
end

63
spec/lib/gitlab/danger/helper_spec.rb
View File

@ -369,6 +369,69 @@ RSpec.describe Gitlab::Danger::Helper do
end
end
describe '#cherry_pick_mr?' do
it 'returns false when `gitlab_helper` is unavailable' do
expect(helper).to receive(:gitlab_helper).and_return(nil)
expect(helper).not_to be_cherry_pick_mr
end
context 'when MR title does not mention a cherry-pick' do
it 'returns false' do
expect(fake_gitlab).to receive(:mr_json)
.and_return('title' => 'Add feature xyz')
expect(helper).not_to be_cherry_pick_mr
end
end
context 'when MR title mentions a cherry-pick' do
[
'Cherry Pick !1234',
'cherry-pick !1234',
'CherryPick !1234'
].each do |mr_title|
it 'returns true' do
expect(fake_gitlab).to receive(:mr_json)
.and_return('title' => mr_title)
expect(helper).to be_cherry_pick_mr
end
end
end
end
describe '#stable_branch?' do
it 'returns false when `gitlab_helper` is unavailable' do
expect(helper).to receive(:gitlab_helper).and_return(nil)
expect(helper).not_to be_stable_branch
end
context 'when MR target branch is not a stable branch' do
it 'returns false' do
expect(fake_gitlab).to receive(:mr_json)
.and_return('target_branch' => 'my-feature-branch')
expect(helper).not_to be_stable_branch
end
end
context 'when MR target branch is a stable branch' do
%w[
13-1-stable-ee
13-1-stable-ee-patch-1
].each do |target_branch|
it 'returns true' do
expect(fake_gitlab).to receive(:mr_json)
.and_return('target_branch' => target_branch)
expect(helper).to be_stable_branch
end
end
end
end
describe '#mr_has_label?' do
it 'returns false when `gitlab_helper` is unavailable' do
expect(helper).to receive(:gitlab_helper).and_return(nil)

147
spec/support/shared_examples/controllers/metrics/dashboard/prometheus_api_proxy_shared_examples.rb Normal file
View File

@ -0,0 +1,147 @@
# frozen_string_literal: true
RSpec.shared_examples_for 'metrics dashboard prometheus api proxy' do
let(:service_params) { [proxyable, 'GET', 'query', expected_params] }
let(:service_result) { { status: :success, body: prometheus_body } }
let(:prometheus_proxy_service) { instance_double(Prometheus::ProxyService) }
let(:proxyable_params) do
{
id: proxyable.id.to_s
}
end
let(:expected_params) do
ActionController::Parameters.new(
prometheus_proxy_params(
proxy_path: 'query',
controller: described_class.controller_path,
action: 'prometheus_proxy'
)
).permit!
end
before do
allow_next_instance_of(Prometheus::ProxyService, *service_params) do |proxy_service|
allow(proxy_service).to receive(:execute).and_return(service_result)
end
end
context 'with valid requests' do
context 'with success result' do
let(:prometheus_body) { '{"status":"success"}' }
let(:prometheus_json_body) { Gitlab::Json.parse(prometheus_body) }
it 'returns prometheus response' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(Prometheus::ProxyService).to have_received(:new).with(*service_params)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to eq(prometheus_json_body)
end
context 'with nil query' do
let(:params_without_query) do
prometheus_proxy_params.except(:query)
end
before do
expected_params.delete(:query)
end
it 'does not raise error' do
get :prometheus_proxy, params: params_without_query
expect(Prometheus::ProxyService).to have_received(:new).with(*service_params)
end
end
end
context 'with nil result' do
let(:service_result) { nil }
it 'returns 204 no_content' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(json_response['status']).to eq(_('processing'))
expect(json_response['message']).to eq(_('Not ready yet. Try again later.'))
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'with 404 result' do
let(:service_result) { { http_status: 404, status: :success, body: '{"body": "value"}' } }
it 'returns body' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(response).to have_gitlab_http_status(:not_found)
expect(json_response['body']).to eq('value')
end
end
context 'with error result' do
context 'with http_status' do
let(:service_result) do
{ http_status: :service_unavailable, status: :error, message: 'error message' }
end
it 'sets the http response status code' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(response).to have_gitlab_http_status(:service_unavailable)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
end
end
context 'without http_status' do
let(:service_result) { { status: :error, message: 'error message' } }
it 'returns bad_request' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['status']).to eq('error')
expect(json_response['message']).to eq('error message')
end
end
end
end
context 'with inappropriate requests' do
let(:prometheus_body) { nil }
context 'without correct permissions' do
let(:user2) { create(:user) }
before do
sign_out(user)
sign_in(user2)
end
it 'returns 404' do
get :prometheus_proxy, params: prometheus_proxy_params
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
context 'with invalid proxyable id' do
let(:prometheus_body) { nil }
it 'returns 404' do
get :prometheus_proxy, params: prometheus_proxy_params(id: proxyable.id + 1)
expect(response).to have_gitlab_http_status(:not_found)
end
end
private
def prometheus_proxy_params(params = {})
{
proxy_path: 'query',
query: '1'
}.merge(proxyable_params).merge(params)
end
end