kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-07-27 06:09:01 +00:00
parent 6108e5b77a
commit a246b5fffa
24 changed files with 237 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/ci/pipeline.rb
View File

@ -393,6 +393,10 @@ module Ci
newest_first(ref: ref).failed.take
end
def self.jobs_count_in_alive_pipelines
created_after(24.hours.ago).alive.joins(:builds).count
end
# Returns a Hash containing the latest pipeline for every given
# commit.
#

10
app/workers/all_queues.yml
View File

@ -1020,6 +1020,7 @@
:idempotent:
:tags:
- :exclude_from_gitlab_com
- :needs_own_queue
- :name: hashed_storage:hashed_storage_project_migrate
:worker_name: HashedStorage::ProjectMigrateWorker
:feature_category: :source_code_management
@ -1030,6 +1031,7 @@
:idempotent:
:tags:
- :exclude_from_gitlab_com
- :needs_own_queue
- :name: hashed_storage:hashed_storage_project_rollback
:worker_name: HashedStorage::ProjectRollbackWorker
:feature_category: :source_code_management
@ -1040,6 +1042,7 @@
:idempotent:
:tags:
- :exclude_from_gitlab_com
- :needs_own_queue
- :name: hashed_storage:hashed_storage_rollbacker
:worker_name: HashedStorage::RollbackerWorker
:feature_category: :source_code_management
@ -1050,6 +1053,7 @@
:idempotent:
:tags:
- :exclude_from_gitlab_com
- :needs_own_queue
- :name: incident_management:clusters_applications_check_prometheus_health
:worker_name: Clusters::Applications::CheckPrometheusHealthWorker
:feature_category: :incident_management
@ -2028,7 +2032,8 @@
:resource_boundary: :unknown
:weight: 2
:idempotent:
:tags: []
:tags:
- :needs_own_queue
- :name: emails_on_push
:worker_name: EmailsOnPushWorker
:feature_category: :source_code_management
@ -2771,7 +2776,8 @@
:resource_boundary: :unknown
:weight: 1
:idempotent:
:tags: []
:tags:
- :needs_own_queue
- :name: snippets_schedule_bulk_repository_shard_moves
:worker_name: Snippets::ScheduleBulkRepositoryShardMovesWorker
:feature_category: :gitaly

3
app/workers/email_receiver_worker.rb
View File

@ -11,6 +11,9 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker
urgency :high
weight 2
# https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application
tags :needs_own_queue
attr_accessor :raw
def perform(raw)

5
app/workers/hashed_storage/migrator_worker.rb
View File

@ -10,7 +10,10 @@ module HashedStorage
queue_namespace :hashed_storage
feature_category :source_code_management
tags :exclude_from_gitlab_com
# Gitlab::HashedStorage::Migrator#migration_pending? depends on the
# queue size of this worker.
tags :exclude_from_gitlab_com, :needs_own_queue
# @param [Integer] start initial ID of the batch
# @param [Integer] finish last ID of the batch

5
app/workers/hashed_storage/project_migrate_worker.rb
View File

@ -10,7 +10,10 @@ module HashedStorage
queue_namespace :hashed_storage
loggable_arguments 1
tags :exclude_from_gitlab_com
# Gitlab::HashedStorage::Migrator#migration_pending? depends on the
# queue size of this worker.
tags :exclude_from_gitlab_com, :needs_own_queue
attr_reader :project_id

5
app/workers/hashed_storage/project_rollback_worker.rb
View File

@ -10,7 +10,10 @@ module HashedStorage
queue_namespace :hashed_storage
loggable_arguments 1
tags :exclude_from_gitlab_com
# Gitlab::HashedStorage::Migrator#rollback_pending? depends on the
# queue size of this worker.
tags :exclude_from_gitlab_com, :needs_own_queue
attr_reader :project_id

5
app/workers/hashed_storage/rollbacker_worker.rb
View File

@ -10,7 +10,10 @@ module HashedStorage
queue_namespace :hashed_storage
feature_category :source_code_management
tags :exclude_from_gitlab_com
# Gitlab::HashedStorage::Migrator#rollback_pending? depends on the
# queue size of this worker.
tags :exclude_from_gitlab_com, :needs_own_queue
# @param [Integer] start initial ID of the batch
# @param [Integer] finish last ID of the batch

3
app/workers/service_desk_email_receiver_worker.rb
View File

@ -8,6 +8,9 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca
feature_category :service_desk
sidekiq_options retry: 3
# https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application
tags :needs_own_queue
def should_perform?
::Gitlab::ServiceDeskEmail.enabled?
end

1
doc/administration/monitoring/prometheus/gitlab_metrics.md
View File

@ -45,6 +45,7 @@ The following metrics are available:
| `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` |
| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` |
| `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` |
| `gitlab_ci_active_jobs` | Histogram | 14.2 | Count of active jobs when pipeline is created | |
| `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | |
| `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` |
| `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` |

39
doc/administration/postgresql/pgbouncer.md
View File

@ -200,6 +200,45 @@ Once you've performed the tasks or procedure, switch back to using PgBouncer:
sudo gitlab-ctl reconfigure
```
## Fine tuning
PgBouncer's default settings suit the majority of installations.
In specific cases you may want to change the performance-specific and resource-specific variables to either increase possible
throughput or to limit resource utilization that could cause memory exhaustion on the database.
You can find the parameters and respective documentation on the [official PgBouncer documentation](https://www.pgbouncer.org/config.html).
Listed below are the most relevant ones and their defaults on an Omnibus GitLab installation:
- `pgbouncer['max_client_conn']` (default: `2048`, depends on server file descriptor limits)
This is the "frontend" pool in PgBouncer: connections from Rails to PgBouncer.
- `pgbouncer['default_pool_size']` (default: `100`)
This is the "backend" pool in PgBouncer: connections from PgBouncer to the database.
The ideal number for `default_pool_size` must be enough to handle all provisioned services that need to access
the database. Each of the listed services below use the following formula to define database pool size:
- `puma` : `max_threads + headroom` (default `14`)
- `max_threads` is configured via: `gitlab['puma']['max_threads']` (default: `4`)
- `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`)
- `sidekiq` : `max_concurrency + 1 + headroom` (default: `61`)
- `max_concurrency` is configured via: `sidekiq['max_concurrency']` (default: `50`)
- `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`)
- `geo-logcursor`: `1+headroom` (default: `11`)
- `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`)
To calculate the `default_pool_size`, multiply the number of instances of `puma`, `sidekiq` and `geo-logcursor` by the
number of connections each can consume as per listed above. The total will be the suggested `default_pool_size`.
If you are using more than one PgBouncer with an internal Load Balancer, you may be able to divide the
`default_pool_size` by the number of instances to guarantee an evenly distributed load between them.
The `pgbouncer['max_client_conn']` is the hard-limit of connections PgBouncer can accept. It's unlikely you will need
to change this. If you are hitting that limit, you may want to consider adding additional PgBouncers with an internal
Load Balancer.
When setting up the limits for a PgBouncer that points to the Geo Tracking Database,
you can likely ignore `puma` from the equation, as it is only accessing that database sporadically.
## Troubleshooting
In case you are experiencing any issues connecting through PgBouncer, the first

5
doc/user/group/index.md
View File

@ -650,8 +650,9 @@ By default, projects in a group can be forked.
Optionally, on [Premium](https://about.gitlab.com/pricing/) or higher tiers,
you can prevent the projects in a group from being forked outside of the current top-level group.
Previously, this setting was available only for groups enforcing group managed
account. This setting will be removed from the SAML setting page, and migrated to the
Previously, this setting was available only for groups enforcing a
[Group Managed Account](saml_sso/group_managed_accounts.md) in SAML.
This setting will be removed from the SAML setting page, and migrated to the
group settings page. In the interim period, both of these settings are taken into consideration.
If even one is set to `true`, then the group does not allow outside forks.

5
lib/gitlab/ci/pipeline/chain/command.rb
View File

@ -97,6 +97,11 @@ module Gitlab
.observe({ source: pipeline.source.to_s }, pipeline.total_size)
end
def observe_jobs_count_in_alive_pipelines
metrics.active_jobs_histogram
.observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines)
end
def increment_pipeline_failure_reason_counter(reason)
metrics.pipeline_failure_reason_counter
.increment(reason: (reason || :unknown_failure).to_s)

1
lib/gitlab/ci/pipeline/chain/sequence.rb
View File

@ -22,6 +22,7 @@ module Gitlab
@command.observe_creation_duration(Time.now - @start)
@command.observe_pipeline_size(@pipeline)
@command.observe_jobs_count_in_alive_pipelines
@pipeline
end

9
lib/gitlab/ci/pipeline/metrics.rb
View File

@ -29,6 +29,15 @@ module Gitlab
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
def self.active_jobs_histogram
name = :gitlab_ci_active_jobs
comment = 'Total amount of active jobs'
labels = { plan: nil }
buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000]
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
def self.pipeline_processing_events_counter
name = :gitlab_ci_pipeline_processing_events_total
comment = 'Total amount of pipeline processing events'

21
locale/gitlab.pot
View File

@ -4130,9 +4130,18 @@ msgid_plural "ApprovalRuleSummary|%{count} approvals required from %{membersCoun
msgstr[0] ""
msgstr[1] ""
msgid "ApprovalRule|%{scanner} +%{additionalScanners} more"
msgstr ""
msgid "ApprovalRule|Add approvers"
msgstr ""
msgid "ApprovalRule|All scanners"
msgstr ""
msgid "ApprovalRule|Apply this approval rule to consider only the selected security scanners."
msgstr ""
msgid "ApprovalRule|Approval rules"
msgstr ""
@ -4151,9 +4160,21 @@ msgstr ""
msgid "ApprovalRule|Name"
msgstr ""
msgid "ApprovalRule|Please select at least one security scanner"
msgstr ""
msgid "ApprovalRule|Rule name"
msgstr ""
msgid "ApprovalRule|Security scanners"
msgstr ""
msgid "ApprovalRule|Select All"
msgstr ""
msgid "ApprovalRule|Select scanners"
msgstr ""
msgid "ApprovalRule|Target branch"
msgstr ""

17
spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb
View File

@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
let_it_be(:user) { create(:user) }
let(:pipeline) { build_stubbed(:ci_pipeline) }
let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new }
let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
let(:first_step) { spy('first step') }
let(:second_step) { spy('second step') }
let(:sequence) { [first_step, second_step] }
@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
expect(histogram).to have_received(:observe)
.with({ source: 'push' }, 0)
end
it 'records active jobs by pipeline plan in a histogram' do
allow(command.metrics)
.to receive(:active_jobs_histogram)
.and_return(histogram)
pipeline = create(:ci_pipeline, project: project, status: :running)
create(:ci_build, :finished, project: project, pipeline: pipeline)
create(:ci_build, :failed, project: project, pipeline: pipeline)
create(:ci_build, :running, project: project, pipeline: pipeline)
subject.build!
expect(histogram).to have_received(:observe)
.with(hash_including(plan: project.actual_plan_name), 3)
end
end
end

2
workhorse/config.toml.example
View File

@ -1,5 +1,7 @@
# alt_document_root = '/home/git/public/assets'
# shutdown_timeout = "60s"
# trusted_cidrs_for_x_forwarded_for = []
# trusted_cidrs_for_propagation = []
[redis]
URL = "unix:/home/git/gitlab/redis/redis.socket"

5
workhorse/config_test.go
View File

@ -30,6 +30,9 @@ func TestConfigFile(t *testing.T) {
data := `
shutdown_timeout = "60s"
trusted_cidrs_for_x_forwarded_for = ["127.0.0.1/8", "192.168.0.1/8"]
trusted_cidrs_for_propagation = ["10.0.0.1/8"]
[redis]
password = "redis password"
[object_storage]
@ -51,6 +54,8 @@ max_scaler_procs = 123
require.Equal(t, "redis password", cfg.Redis.Password)
require.Equal(t, "test provider", cfg.ObjectStorageCredentials.Provider)
require.Equal(t, uint32(123), cfg.ImageResizerConfig.MaxScalerProcs, "image resizer max_scaler_procs")
require.Equal(t, []string{"127.0.0.1/8", "192.168.0.1/8"}, cfg.TrustedCIDRsForXForwardedFor)
require.Equal(t, []string{"10.0.0.1/8"}, cfg.TrustedCIDRsForPropagation)
require.Equal(t, 60*time.Second, cfg.ShutdownTimeout.Duration)
}

5
workhorse/go.mod
View File

@ -7,7 +7,7 @@ require (
github.com/BurntSushi/toml v0.3.1
github.com/FZambia/sentinel v1.0.0
github.com/alecthomas/chroma v0.7.3
github.com/aws/aws-sdk-go v1.36.1
github.com/aws/aws-sdk-go v1.37.0
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.2
@ -29,7 +29,7 @@ require (
github.com/smartystreets/goconvey v1.6.4
github.com/stretchr/testify v1.7.0
gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1
gitlab.com/gitlab-org/labkit v1.4.0
gitlab.com/gitlab-org/labkit v1.6.0
gocloud.dev v0.21.1-0.20201223184910-5094f54ed8bb
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 // indirect
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
@ -37,5 +37,6 @@ require (
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4
golang.org/x/tools v0.1.0
google.golang.org/grpc v1.37.0
gopkg.in/DataDog/dd-trace-go.v1 v1.31.0 // indirect
honnef.co/go/tools v0.1.3
)

24
workhorse/go.sum
View File

@ -19,6 +19,7 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb
cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko=
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
@ -46,6 +47,8 @@ cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lU
cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho=
contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc=
contrib.go.opencensus.io/exporter/stackdriver v0.13.8 h1:lIFYmQsqejvlq+GobFUbC5F0prD5gvhP6r0gWLZRDq4=
contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ=
contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
@ -145,8 +148,9 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.36.1 h1:rDgSL20giXXu48Ycx6Qa4vWaNTVTltUl6vA73ObCSVk=
github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk=
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -158,6 +162,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI=
@ -309,8 +314,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -379,6 +385,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo=
@ -783,8 +790,8 @@ gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h
gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM=
gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029/go.mod h1:SNfxkfUwVNECgtmluVayv0GWFgEjjBs5AzgsowPQuo0=
gitlab.com/gitlab-org/labkit v1.0.0/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U=
gitlab.com/gitlab-org/labkit v1.4.0 h1:KZTEylusrFmqLXSzE5bHfBf7/xI2NLnsyoRgB7I7Oh8=
gitlab.com/gitlab-org/labkit v1.4.0/go.mod h1:4YbseTLUD7g4pPSylV57Hpyf7N3hbbxdx8K81//U/XM=
gitlab.com/gitlab-org/labkit v1.6.0 h1:Qgk+W+N0cujGBmZSjMqvM+4qIEjl7VgIK4nxlQO0RlA=
gitlab.com/gitlab-org/labkit v1.6.0/go.mod h1:1ZuVZpjSpCKUgjLx8P6jzkkQFxJI1thUKr6yKV3p0vY=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
@ -796,6 +803,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@ -921,6 +929,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
@ -934,6 +943,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
@ -949,6 +959,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1103,6 +1114,7 @@ golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1139,6 +1151,7 @@ google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQW
google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
google.golang.org/api v0.37.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
@ -1196,6 +1209,8 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@ -1245,6 +1260,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/DataDog/dd-trace-go.v1 v1.7.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
gopkg.in/DataDog/dd-trace-go.v1 v1.30.0/go.mod h1:SnKViq44dv/0gjl9RpkP0Y2G3BJSRkp6eYdCSu39iI8=
gopkg.in/DataDog/dd-trace-go.v1 v1.31.0 h1:ouY+DNlRTckk63TNh468tPWBC21qBZPniVQXQs0iq10=
gopkg.in/DataDog/dd-trace-go.v1 v1.31.0/go.mod h1:SnKViq44dv/0gjl9RpkP0Y2G3BJSRkp6eYdCSu39iI8=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

40
workhorse/internal/config/config.go
View File

@ -85,25 +85,27 @@ type ImageResizerConfig struct {
}
type Config struct {
Redis *RedisConfig `toml:"redis"`
Backend *url.URL `toml:"-"`
CableBackend *url.URL `toml:"-"`
Version string `toml:"-"`
DocumentRoot string `toml:"-"`
DevelopmentMode bool `toml:"-"`
Socket string `toml:"-"`
CableSocket string `toml:"-"`
ProxyHeadersTimeout time.Duration `toml:"-"`
APILimit uint `toml:"-"`
APIQueueLimit uint `toml:"-"`
APIQueueTimeout time.Duration `toml:"-"`
APICILongPollingDuration time.Duration `toml:"-"`
ObjectStorageConfig ObjectStorageConfig `toml:"-"`
ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
PropagateCorrelationID bool `toml:"-"`
ImageResizerConfig ImageResizerConfig `toml:"image_resizer"`
AltDocumentRoot string `toml:"alt_document_root"`
ShutdownTimeout TomlDuration `toml:"shutdown_timeout"`
Redis *RedisConfig `toml:"redis"`
Backend *url.URL `toml:"-"`
CableBackend *url.URL `toml:"-"`
Version string `toml:"-"`
DocumentRoot string `toml:"-"`
DevelopmentMode bool `toml:"-"`
Socket string `toml:"-"`
CableSocket string `toml:"-"`
ProxyHeadersTimeout time.Duration `toml:"-"`
APILimit uint `toml:"-"`
APIQueueLimit uint `toml:"-"`
APIQueueTimeout time.Duration `toml:"-"`
APICILongPollingDuration time.Duration `toml:"-"`
ObjectStorageConfig ObjectStorageConfig `toml:"-"`
ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
PropagateCorrelationID bool `toml:"-"`
ImageResizerConfig ImageResizerConfig `toml:"image_resizer"`
AltDocumentRoot string `toml:"alt_document_root"`
ShutdownTimeout TomlDuration `toml:"shutdown_timeout"`
TrustedCIDRsForXForwardedFor []string `toml:"trusted_cidrs_for_x_forwarded_for"`
TrustedCIDRsForPropagation []string `toml:"trusted_cidrs_for_propagation"`
}
var DefaultImageResizerConfig = ImageResizerConfig{

6
workhorse/internal/upstream/upstream.go
View File

@ -87,6 +87,12 @@ func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback
if cfg.PropagateCorrelationID {
correlationOpts = append(correlationOpts, correlation.WithPropagation())
}
if cfg.TrustedCIDRsForPropagation != nil {
correlationOpts = append(correlationOpts, correlation.WithCIDRsTrustedForPropagation(cfg.TrustedCIDRsForPropagation))
}
if cfg.TrustedCIDRsForXForwardedFor != nil {
correlationOpts = append(correlationOpts, correlation.WithCIDRsTrustedForXForwardedFor(cfg.TrustedCIDRsForXForwardedFor))
}
handler := correlation.InjectCorrelationID(&up, correlationOpts...)
// TODO: move to LabKit https://gitlab.com/gitlab-org/gitlab/-/issues/324823

2
workhorse/main.go
View File

@ -152,6 +152,8 @@ func buildConfig(arg0 string, args []string) (*bootConfig, *config.Config, error
cfg.ImageResizerConfig = cfgFromFile.ImageResizerConfig
cfg.AltDocumentRoot = cfgFromFile.AltDocumentRoot
cfg.ShutdownTimeout = cfgFromFile.ShutdownTimeout
cfg.TrustedCIDRsForXForwardedFor = cfgFromFile.TrustedCIDRsForXForwardedFor
cfg.TrustedCIDRsForPropagation = cfgFromFile.TrustedCIDRsForPropagation
return boot, cfg, nil
}

53
workhorse/main_test.go
View File

@ -611,16 +611,53 @@ func TestPropagateCorrelationIdHeader(t *testing.T) {
defer ts.Close()
testCases := []struct {
desc string
propagateCorrelationID bool
desc string
propagateCorrelationID bool
xffHeader string
trustedCIDRsForPropagation []string
trustedCIDRsForXForwardedFor []string
propagationExpected bool
}{
{
desc: "propagateCorrelatedId is true",
propagateCorrelationID: true,
propagationExpected: true,
},
{
desc: "propagateCorrelatedId is false",
propagateCorrelationID: false,
propagationExpected: false,
},
{
desc: "propagation with trusted propagation CIDR",
propagateCorrelationID: true,
// Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x
trustedCIDRsForPropagation: []string{"127.0.0.1/8"},
propagationExpected: true,
},
{
desc: "propagation with trusted propagation and X-Forwarded-For CIDRs",
propagateCorrelationID: true,
// Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x
xffHeader: "1.2.3.4, 127.0.0.1",
trustedCIDRsForPropagation: []string{"1.2.3.4/32"},
trustedCIDRsForXForwardedFor: []string{"127.0.0.1/32", "192.168.0.1/32"},
propagationExpected: true,
},
{
desc: "propagation not active with invalid propagation CIDR",
propagateCorrelationID: true,
trustedCIDRsForPropagation: []string{"asdf"},
propagationExpected: false,
},
{
desc: "propagation with invalid X-Forwarded-For CIDR",
propagateCorrelationID: true,
// Assumes HTTP connection's RemoteAddr will be 127.0.0.1:x
xffHeader: "1.2.3.4, 127.0.0.1",
trustedCIDRsForPropagation: []string{"1.2.3.4/32"},
trustedCIDRsForXForwardedFor: []string{"bad"},
propagationExpected: false,
},
}
@ -628,19 +665,27 @@ func TestPropagateCorrelationIdHeader(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
upstreamConfig := newUpstreamConfig(ts.URL)
upstreamConfig.PropagateCorrelationID = tc.propagateCorrelationID
upstreamConfig.TrustedCIDRsForPropagation = tc.trustedCIDRsForPropagation
upstreamConfig.TrustedCIDRsForXForwardedFor = tc.trustedCIDRsForXForwardedFor
ws := startWorkhorseServerWithConfig(upstreamConfig)
defer ws.Close()
resource := "/api/v3/projects/123/repository/not/special"
propagatedRequestId := "Propagated-RequestId-12345678"
resp, _ := httpGet(t, ws.URL+resource, map[string]string{"X-Request-Id": propagatedRequestId})
headers := map[string]string{"X-Request-Id": propagatedRequestId}
if tc.xffHeader != "" {
headers["X-Forwarded-For"] = tc.xffHeader
}
resp, _ := httpGet(t, ws.URL+resource, headers)
requestIds := resp.Header["X-Request-Id"]
require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource)
require.Equal(t, 1, len(requestIds), "GET %q: One X-Request-Id present", resource)
if tc.propagateCorrelationID {
if tc.propagationExpected {
require.Contains(t, requestIds, propagatedRequestId, "GET %q: Has X-Request-Id %s present", resource, propagatedRequestId)
} else {
require.NotContains(t, requestIds, propagatedRequestId, "GET %q: X-Request-Id not propagated")