kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-08-31 06:12:36 +00:00
parent a5bd90f43b
commit 136651d7cb
24 changed files with 225 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/controllers/projects_controller.rb
View File

@ -12,6 +12,8 @@ class ProjectsController < Projects::ApplicationController
include SourcegraphDecorator include SourcegraphDecorator
include PlanningHierarchy include PlanningHierarchy
REFS_LIMIT = 100
prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:rss) } prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:rss) }
around_action :allow_gitaly_ref_name_caching, only: [:index, :show] around_action :allow_gitaly_ref_name_caching, only: [:index, :show]
@ -309,6 +311,8 @@ class ProjectsController < Projects::ApplicationController
find_tags = true find_tags = true
find_commits = true find_commits = true
use_gitaly_pagination = Feature.enabled?(:use_gitaly_pagination_for_refs, @project)
unless find_refs.nil? unless find_refs.nil?
find_branches = find_refs.include?('branches') find_branches = find_refs.include?('branches')
find_tags = find_refs.include?('tags') find_tags = find_refs.include?('tags')
@ -318,13 +322,21 @@ class ProjectsController < Projects::ApplicationController
options = {} options = {}
if find_branches if find_branches
branches = BranchesFinder.new(@repository, refs_params).execute.take(100).map(&:name) branches = BranchesFinder.new(@repository, refs_params.merge(per_page: REFS_LIMIT))
.execute(gitaly_pagination: use_gitaly_pagination)
.take(REFS_LIMIT)
.map(&:name)
options['Branches'] = branches options['Branches'] = branches
end end
if find_tags && @repository.tag_count.nonzero? if find_tags && @repository.tag_count.nonzero?
tags = TagsFinder.new(@repository, refs_params).execute tags = TagsFinder.new(@repository, refs_params.merge(per_page: REFS_LIMIT))
options['Tags'] = tags.take(100).map(&:name) .execute(gitaly_pagination: use_gitaly_pagination)
.take(REFS_LIMIT)
.map(&:name)
options['Tags'] = tags
end end
# If reference is commit id - we should add it to branch/tag selectbox # If reference is commit id - we should add it to branch/tag selectbox

8
config/feature_flags/development/use_gitaly_pagination_for_refs.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: use_gitaly_pagination_for_refs
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96448
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/372049
milestone: '15.4'
type: development
group: group::source code
default_enabled: false

18
db/post_migrate/20220825061250_drop_tmp_index_todos_attention_request_action_idx.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class DropTmpIndexTodosAttentionRequestActionIdx < Gitlab::Database::Migration[2.0]
INDEX_NAME = "tmp_index_todos_attention_request_action"
ATTENTION_REQUESTED = 10
disable_ddl_transaction!
def up
remove_concurrent_index_by_name :todos, INDEX_NAME
end
def down
add_concurrent_index :todos, [:id],
where: "action = #{ATTENTION_REQUESTED}",
name: INDEX_NAME
end
end

1
db/schema_migrations/20220825061250 Normal file
View File

@ -0,0 +1 @@
0338843ad56b423559e613f00df205122b4f6db194cf49712b2ff46b2ad030e0

2
db/structure.sql
View File

@ -30668,8 +30668,6 @@ CREATE INDEX tmp_index_project_statistics_cont_registry_size ON project_statisti
CREATE INDEX tmp_index_system_note_metadata_on_id_where_task ON system_note_metadata USING btree (id, action) WHERE ((action)::text = 'task'::text); CREATE INDEX tmp_index_system_note_metadata_on_id_where_task ON system_note_metadata USING btree (id, action) WHERE ((action)::text = 'task'::text);
CREATE INDEX tmp_index_todos_attention_request_action ON todos USING btree (id) WHERE (action = 10);
CREATE INDEX tmp_index_vulnerability_occurrences_on_id_and_scanner_id ON vulnerability_occurrences USING btree (id, scanner_id) WHERE (report_type = ANY (ARRAY[7, 99])); CREATE INDEX tmp_index_vulnerability_occurrences_on_id_and_scanner_id ON vulnerability_occurrences USING btree (id, scanner_id) WHERE (report_type = ANY (ARRAY[7, 99]));
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name); CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);

2
doc/administration/index.md
View File

@ -82,7 +82,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
#### Customizing GitLab appearance #### Customizing GitLab appearance
- [Header logo](../user/admin_area/appearance.md#navigation-bar): Change the logo on all pages and email headers. - [Header logo](../user/admin_area/appearance.md#top-bar): Change the logo on all pages and email headers.
- [Favicon](../user/admin_area/appearance.md#favicon): Change the default favicon to your own logo. - [Favicon](../user/admin_area/appearance.md#favicon): Change the default favicon to your own logo.
- [Branded login page](../user/admin_area/appearance.md#sign-in--sign-up-pages): Customize the login page with your own logo, title, and description. - [Branded login page](../user/admin_area/appearance.md#sign-in--sign-up-pages): Customize the login page with your own logo, title, and description.
- ["New Project" page](../user/admin_area/appearance.md#new-project-pages): Customize the text to be displayed on the page that opens whenever your users create a new project. - ["New Project" page](../user/admin_area/appearance.md#new-project-pages): Customize the text to be displayed on the page that opens whenever your users create a new project.

4
doc/user/admin_area/appearance.md
View File

@ -13,9 +13,9 @@ of GitLab. To access these settings:
1. On the top bar, select **Menu > Admin**. 1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Appearance**. 1. On the left sidebar, select **Settings > Appearance**.
## Navigation bar ## Top bar
By default, the navigation bar has the GitLab logo, but this can be customized with By default, the **top bar** has the GitLab logo, but this can be customized with
any image desired. It is optimized for images 28px high (any width), but any image can be any image desired. It is optimized for images 28px high (any width), but any image can be
used (less than 1 MB) and it is automatically resized. used (less than 1 MB) and it is automatically resized.

2
doc/user/admin_area/settings/email.md
View File

@ -11,7 +11,7 @@ You can customize some of the content in emails sent from your GitLab instance.
## Custom logo ## Custom logo
The logo in the header of some emails can be customized, see the [logo customization section](../appearance.md#navigation-bar). The logo in the header of some emails can be customized, see the [logo customization section](../appearance.md#top-bar).
## Include author name in email notification email body **(PREMIUM SELF)** ## Include author name in email notification email body **(PREMIUM SELF)**

8
package.json
View File

@ -52,8 +52,8 @@
"@codesandbox/sandpack-client": "^1.2.2", "@codesandbox/sandpack-client": "^1.2.2",
"@gitlab/at.js": "1.5.7", "@gitlab/at.js": "1.5.7",
"@gitlab/favicon-overlay": "2.0.0", "@gitlab/favicon-overlay": "2.0.0",
"@gitlab/svgs": "3.2.0", "@gitlab/svgs": "3.3.0",
"@gitlab/ui": "43.9.1", "@gitlab/ui": "43.9.3",
"@gitlab/visual-review-tools": "1.7.3", "@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20220815034418", "@gitlab/web-ide": "0.0.1-dev-20220815034418",
"@rails/actioncable": "6.1.4-7", "@rails/actioncable": "6.1.4-7",
@ -244,10 +244,10 @@
"sass": "^1.49.9", "sass": "^1.49.9",
"stylelint": "^14.9.1", "stylelint": "^14.9.1",
"timezone-mock": "^1.0.8", "timezone-mock": "^1.0.8",
"webpack-dev-server": "4.10.0", "webpack-dev-server": "4.10.1",
"xhr-mock": "^2.5.1", "xhr-mock": "^2.5.1",
"yarn-check-webpack-plugin": "^1.2.0", "yarn-check-webpack-plugin": "^1.2.0",
"yarn-deduplicate": "^5.0.2" "yarn-deduplicate": "^6.0.0"
}, },
"blockedDependencies": { "blockedDependencies": {
"bootstrap-vue": "https://docs.gitlab.com/ee/development/fe_guide/dependencies.html#bootstrapvue" "bootstrap-vue": "https://docs.gitlab.com/ee/development/fe_guide/dependencies.html#bootstrapvue"

9
qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb
View File

@ -88,14 +88,19 @@ module QA
let(:gh_issue_comments) do let(:gh_issue_comments) do
logger.debug("= Fetching issue comments =") logger.debug("= Fetching issue comments =")
github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url) # use base html url as key # use base html url as key
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url)
end end
end end
let(:gh_pr_comments) do let(:gh_pr_comments) do
logger.debug("= Fetching pr comments =") logger.debug("= Fetching pr comments =")
github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url) # use base html url as key # use base html url as key
hash[c.html_url.gsub(/\#\S+/, "")] << c.body
# some suggestions can contain extra whitespaces which gitlab will remove
&.gsub(/suggestion\s+\r/, "suggestion\r")
&.gsub(gh_link_pattern, dummy_url)
end end
end end

2
qa/qa/specs/features/api/3_create/repository/files_spec.rb
View File

@ -103,7 +103,7 @@ module QA
expect(response.headers[:expires]).to eq("Fri, 01 Jan 1990 00:00:00 GMT") expect(response.headers[:expires]).to eq("Fri, 01 Jan 1990 00:00:00 GMT")
expect(response.headers[:content_disposition]).to include("attachment") expect(response.headers[:content_disposition]).to include("attachment")
expect(response.headers[:content_disposition]).not_to include("inline") expect(response.headers[:content_disposition]).not_to include("inline")
expect(response.headers[:content_type]).to include("application/octet-stream") expect(response.headers[:content_type]).to include("image/svg+xml")
end end
delete_project_request = Runtime::API::Request.new(@api_client, "/projects/#{sanitized_project_path}") delete_project_request = Runtime::API::Request.new(@api_client, "/projects/#{sanitized_project_path}")

6
qa/qa/specs/features/browser_ui/1_manage/project/dashboard_images_spec.rb
View File

@ -20,8 +20,10 @@ module QA
Flow::Login.sign_in(as: user) Flow::Login.sign_in(as: user)
Page::Dashboard::Welcome.perform do |welcome| Page::Dashboard::Welcome.perform do |welcome|
expect(welcome).to have_welcome_title("Welcome to GitLab") Support::Waiter.wait_until(sleep_interval: 2, max_duration: 60, reload_page: page,
retry_on_exception: true) do
expect(welcome).to have_welcome_title("Welcome to GitLab")
end
# This would be better if it were a visual validation test # This would be better if it were a visual validation test
expect(welcome).to have_loaded_all_images expect(welcome).to have_loaded_all_images
end end

34
spec/controllers/projects_controller_spec.rb
View File

@ -1217,6 +1217,40 @@ RSpec.describe ProjectsController do
expect(json_response["Commits"]).to include("123456") expect(json_response["Commits"]).to include("123456")
end end
it 'uses gitaly pagination' do
expected_params = ActionController::Parameters.new(ref: '123456', per_page: 100).permit!
expect_next_instance_of(BranchesFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: true).and_call_original
end
expect_next_instance_of(TagsFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: true).and_call_original
end
get :refs, params: { namespace_id: project.namespace, id: project, ref: "123456" }
end
context 'when use_gitaly_pagination_for_refs is disabled' do
before do
stub_feature_flags(use_gitaly_pagination_for_refs: false)
end
it 'does not use gitaly pagination' do
expected_params = ActionController::Parameters.new(ref: '123456', per_page: 100).permit!
expect_next_instance_of(BranchesFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: false).and_call_original
end
expect_next_instance_of(TagsFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: false).and_call_original
end
get :refs, params: { namespace_id: project.namespace, id: project, ref: "123456" }
end
end
context 'when gitaly is unavailable' do context 'when gitaly is unavailable' do
before do before do
expect_next_instance_of(TagsFinder) do |finder| expect_next_instance_of(TagsFinder) do |finder|

4
workhorse/go.mod
View File

@ -26,7 +26,7 @@ require (
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.0
github.com/smartystreets/goconvey v1.7.2 github.com/smartystreets/goconvey v1.7.2
github.com/stretchr/testify v1.8.0 github.com/stretchr/testify v1.8.0
gitlab.com/gitlab-org/gitaly/v15 v15.2.2 gitlab.com/gitlab-org/gitaly/v15 v15.3.2
gitlab.com/gitlab-org/golang-archive-zip v0.1.1 gitlab.com/gitlab-org/golang-archive-zip v0.1.1
gitlab.com/gitlab-org/labkit v1.16.0 gitlab.com/gitlab-org/labkit v1.16.0
gocloud.dev v0.25.0 gocloud.dev v0.25.0
@ -74,7 +74,7 @@ require (
github.com/google/wire v0.5.0 // indirect github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/googleapis/gax-go/v2 v2.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/hashicorp/yamux v0.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/kr/text v0.2.0 // indirect github.com/kr/text v0.2.0 // indirect

14
workhorse/go.sum
View File

@ -639,8 +639,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY= github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -1125,11 +1125,11 @@ gitlab.com/gitlab-org/gitaly v1.68.0 h1:VlcJs1+PrhW7lqJUU7Fh1q8FMJujmbbivdfde/cw
gitlab.com/gitlab-org/gitaly v1.68.0/go.mod h1:/pCsB918Zu5wFchZ9hLYin9WkJ2yQqdVNz0zlv5HbXg= gitlab.com/gitlab-org/gitaly v1.68.0/go.mod h1:/pCsB918Zu5wFchZ9hLYin9WkJ2yQqdVNz0zlv5HbXg=
gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1/go.mod h1:4Cz8tOAyueSZX5o6gYum1F/unupaOclxqETPcg4ODvQ= gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1/go.mod h1:4Cz8tOAyueSZX5o6gYum1F/unupaOclxqETPcg4ODvQ=
gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc5.0.20220329111719-51da8bc17059/go.mod h1:uX1qhFKBDuPqATlpMcFL2dKDiX8D/tbUg7CYWx7OXt4= gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc5.0.20220329111719-51da8bc17059/go.mod h1:uX1qhFKBDuPqATlpMcFL2dKDiX8D/tbUg7CYWx7OXt4=
gitlab.com/gitlab-org/gitaly/v15 v15.2.2 h1:/hSbAhBqRrT6Epc35k83qFwwVbKottNY6wDFr+5DYQo= gitlab.com/gitlab-org/gitaly/v15 v15.3.2 h1:H8NoBZil23mZ8Y31XdPWCNp27m2nfFGV+z/pQd5Rsq4=
gitlab.com/gitlab-org/gitaly/v15 v15.2.2/go.mod h1:WjitFL44l9ovitGC4OvSuGwfeq0VpHUbHS6sDw13LV8= gitlab.com/gitlab-org/gitaly/v15 v15.3.2/go.mod h1:DPO/d7DnhJ0TTcL6UZNJ6mcRHhdBPg+f/iBA+LEKEq0=
gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h1:5QSTbpAHY2v0iIH5uHh2KA9w7sPUqPmnLjDApI/sv1U= gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h1:5QSTbpAHY2v0iIH5uHh2KA9w7sPUqPmnLjDApI/sv1U=
gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20210720163109-50da611814d2/go.mod h1:QWDYBwuy24qGMandtCngLRPzFgnGPg6LSNoJWPKmJMc= gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20210720163109-50da611814d2/go.mod h1:QWDYBwuy24qGMandtCngLRPzFgnGPg6LSNoJWPKmJMc=
gitlab.com/gitlab-org/gitlab-shell/v14 v14.8.0/go.mod h1:Z1S5MihpEmtA7GDXGsU0kUf1nzm7zr8w6bP+uXRnxaw= gitlab.com/gitlab-org/gitlab-shell/v14 v14.10.0/go.mod h1:ynuwa2oLLQSOs6Ss6RpLDkAQuFUNreI3NdEOJxfh1ac=
gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE= gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE=
gitlab.com/gitlab-org/golang-archive-zip v0.1.1/go.mod h1:ZDtqpWPGPB9qBuZnZDrKQjIdJtkN7ZAoVwhT6H2o2kE= gitlab.com/gitlab-org/golang-archive-zip v0.1.1/go.mod h1:ZDtqpWPGPB9qBuZnZDrKQjIdJtkN7ZAoVwhT6H2o2kE=
gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM= gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM=
@ -1137,8 +1137,6 @@ gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029/go.mod h1:SNfxkf
gitlab.com/gitlab-org/labkit v1.0.0/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U= gitlab.com/gitlab-org/labkit v1.0.0/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U=
gitlab.com/gitlab-org/labkit v1.4.1/go.mod h1:x5JO5uvdX4t6e/TZXLXZnFL5AcKz2uLLd3uKXZcuO4k= gitlab.com/gitlab-org/labkit v1.4.1/go.mod h1:x5JO5uvdX4t6e/TZXLXZnFL5AcKz2uLLd3uKXZcuO4k=
gitlab.com/gitlab-org/labkit v1.5.0/go.mod h1:1ZuVZpjSpCKUgjLx8P6jzkkQFxJI1thUKr6yKV3p0vY= gitlab.com/gitlab-org/labkit v1.5.0/go.mod h1:1ZuVZpjSpCKUgjLx8P6jzkkQFxJI1thUKr6yKV3p0vY=
gitlab.com/gitlab-org/labkit v1.14.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
gitlab.com/gitlab-org/labkit v1.15.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
gitlab.com/gitlab-org/labkit v1.16.0 h1:Vm3NAMZ8RqAunXlvPWby3GJ2R35vsYGP6Uu0YjyMIlY= gitlab.com/gitlab-org/labkit v1.16.0 h1:Vm3NAMZ8RqAunXlvPWby3GJ2R35vsYGP6Uu0YjyMIlY=
gitlab.com/gitlab-org/labkit v1.16.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc= gitlab.com/gitlab-org/labkit v1.16.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@ -1477,7 +1475,6 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@ -1782,7 +1779,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=

116
workhorse/internal/redis/keywatcher.go
View File

@ -15,17 +15,27 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/log" "gitlab.com/gitlab-org/gitlab/workhorse/internal/log"
) )
var ( type KeyWatcher struct {
keyWatcher = make(map[string][]chan string) mu sync.Mutex
keyWatcherMutex sync.Mutex subscribers map[string][]chan string
shutdown = make(chan struct{}) shutdown chan struct{}
redisReconnectTimeout = backoff.Backoff{ reconnectBackoff backoff.Backoff
//These are the defaults }
Min: 100 * time.Millisecond,
Max: 60 * time.Second, func NewKeyWatcher() *KeyWatcher {
Factor: 2, return &KeyWatcher{
Jitter: true, subscribers: make(map[string][]chan string),
shutdown: make(chan struct{}),
reconnectBackoff: backoff.Backoff{
Min: 100 * time.Millisecond,
Max: 60 * time.Second,
Factor: 2,
Jitter: true,
},
} }
}
var (
keyWatchers = promauto.NewGauge( keyWatchers = promauto.NewGauge(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: "gitlab_workhorse_keywatcher_keywatchers", Name: "gitlab_workhorse_keywatcher_keywatchers",
@ -57,15 +67,9 @@ const (
keySubChannel = "workhorse:notifications" keySubChannel = "workhorse:notifications"
) )
// KeyChan holds a key and a channel
type KeyChan struct {
Key string
Chan chan string
}
func countAction(action string) { totalActions.WithLabelValues(action).Add(1) } func countAction(action string) { totalActions.WithLabelValues(action).Add(1) }
func processInner(conn redis.Conn) error { func (kw *KeyWatcher) receivePubSubStream(conn redis.Conn) error {
defer conn.Close() defer conn.Close()
psc := redis.PubSubConn{Conn: conn} psc := redis.PubSubConn{Conn: conn}
if err := psc.Subscribe(keySubChannel); err != nil { if err := psc.Subscribe(keySubChannel); err != nil {
@ -84,7 +88,7 @@ func processInner(conn redis.Conn) error {
log.WithError(fmt.Errorf("keywatcher: invalid notification: %q", dataStr)).Error() log.WithError(fmt.Errorf("keywatcher: invalid notification: %q", dataStr)).Error()
continue continue
} }
notifyChanWatchers(msg[0], msg[1]) kw.notifySubscribers(msg[0], msg[1])
case error: case error:
log.WithError(fmt.Errorf("keywatcher: pubsub receive: %v", v)).Error() log.WithError(fmt.Errorf("keywatcher: pubsub receive: %v", v)).Error()
// Intermittent error, return nil so that it doesn't wait before reconnect // Intermittent error, return nil so that it doesn't wait before reconnect
@ -109,45 +113,42 @@ func dialPubSub(dialer redisDialerFunc) (redis.Conn, error) {
return conn, nil return conn, nil
} }
// Process redis subscriptions func (kw *KeyWatcher) Process() {
//
// NOTE: There Can Only Be One!
func Process() {
log.Info("keywatcher: starting process loop") log.Info("keywatcher: starting process loop")
for { for {
conn, err := dialPubSub(workerDialFunc) conn, err := dialPubSub(workerDialFunc)
if err != nil { if err != nil {
log.WithError(fmt.Errorf("keywatcher: %v", err)).Error() log.WithError(fmt.Errorf("keywatcher: %v", err)).Error()
time.Sleep(redisReconnectTimeout.Duration()) time.Sleep(kw.reconnectBackoff.Duration())
continue continue
} }
redisReconnectTimeout.Reset() kw.reconnectBackoff.Reset()
if err = processInner(conn); err != nil { if err = kw.receivePubSubStream(conn); err != nil {
log.WithError(fmt.Errorf("keywatcher: process loop: %v", err)).Error() log.WithError(fmt.Errorf("keywatcher: receivePubSubStream: %v", err)).Error()
} }
} }
} }
func Shutdown() { func (kw *KeyWatcher) Shutdown() {
log.Info("keywatcher: shutting down") log.Info("keywatcher: shutting down")
keyWatcherMutex.Lock() kw.mu.Lock()
defer keyWatcherMutex.Unlock() defer kw.mu.Unlock()
select { select {
case <-shutdown: case <-kw.shutdown:
// already closed // already closed
default: default:
close(shutdown) close(kw.shutdown)
} }
} }
func notifyChanWatchers(key, value string) { func (kw *KeyWatcher) notifySubscribers(key, value string) {
keyWatcherMutex.Lock() kw.mu.Lock()
defer keyWatcherMutex.Unlock() defer kw.mu.Unlock()
chanList, ok := keyWatcher[key] chanList, ok := kw.subscribers[key]
if !ok { if !ok {
countAction("drop-message") countAction("drop-message")
return return
@ -158,38 +159,38 @@ func notifyChanWatchers(key, value string) {
c <- value c <- value
keyWatchers.Dec() keyWatchers.Dec()
} }
delete(keyWatcher, key) delete(kw.subscribers, key)
} }
func addKeyChan(kc *KeyChan) { func (kw *KeyWatcher) addSubscription(key string, notify chan string) {
keyWatcherMutex.Lock() kw.mu.Lock()
defer keyWatcherMutex.Unlock() defer kw.mu.Unlock()
keyWatcher[kc.Key] = append(keyWatcher[kc.Key], kc.Chan) kw.subscribers[key] = append(kw.subscribers[key], notify)
keyWatchers.Inc() keyWatchers.Inc()
if len(keyWatcher[kc.Key]) == 1 { if len(kw.subscribers[key]) == 1 {
countAction("create-subscription") countAction("create-subscription")
} }
} }
func delKeyChan(kc *KeyChan) { func (kw *KeyWatcher) delSubscription(key string, notify chan string) {
keyWatcherMutex.Lock() kw.mu.Lock()
defer keyWatcherMutex.Unlock() defer kw.mu.Unlock()
chans, ok := keyWatcher[kc.Key] chans, ok := kw.subscribers[key]
if !ok { if !ok {
return return
} }
for i, c := range chans { for i, c := range chans {
if kc.Chan == c { if notify == c {
keyWatcher[kc.Key] = append(chans[:i], chans[i+1:]...) kw.subscribers[key] = append(chans[:i], chans[i+1:]...)
keyWatchers.Dec() keyWatchers.Dec()
break break
} }
} }
if len(keyWatcher[kc.Key]) == 0 { if len(kw.subscribers[key]) == 0 {
delete(keyWatcher, kc.Key) delete(kw.subscribers, key)
countAction("delete-subscription") countAction("delete-subscription")
} }
} }
@ -209,15 +210,10 @@ const (
WatchKeyStatusNoChange WatchKeyStatusNoChange
) )
// WatchKey waits for a key to be updated or expired func (kw *KeyWatcher) WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error) {
func WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error) { notify := make(chan string, 1)
kw := &KeyChan{ kw.addSubscription(key, notify)
Key: key, defer kw.delSubscription(key, notify)
Chan: make(chan string, 1),
}
addKeyChan(kw)
defer delKeyChan(kw)
currentValue, err := GetString(key) currentValue, err := GetString(key)
if errors.Is(err, redis.ErrNil) { if errors.Is(err, redis.ErrNil) {
@ -230,10 +226,10 @@ func WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error)
} }
select { select {
case <-shutdown: case <-kw.shutdown:
log.WithFields(log.Fields{"key": key}).Info("stopping watch due to shutdown") log.WithFields(log.Fields{"key": key}).Info("stopping watch due to shutdown")
return WatchKeyStatusNoChange, nil return WatchKeyStatusNoChange, nil
case currentValue := <-kw.Chan: case currentValue := <-notify:
if currentValue == "" { if currentValue == "" {
return WatchKeyStatusNoChange, fmt.Errorf("keywatcher: redis GET failed") return WatchKeyStatusNoChange, fmt.Errorf("keywatcher: redis GET failed")
} }

62
workhorse/internal/redis/keywatcher_test.go
View File

@ -38,20 +38,14 @@ func createUnsubscribeMessage(key string) []interface{} {
} }
} }
func countWatchers(key string) int { func (kw *KeyWatcher) countSubscribers(key string) int {
keyWatcherMutex.Lock() kw.mu.Lock()
defer keyWatcherMutex.Unlock() defer kw.mu.Unlock()
return len(keyWatcher[key]) return len(kw.subscribers[key])
}
func deleteWatchers(key string) {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
delete(keyWatcher, key)
} }
// Forces a run of the `Process` loop against a mock PubSubConn. // Forces a run of the `Process` loop against a mock PubSubConn.
func processMessages(numWatchers int, value string) { func (kw *KeyWatcher) processMessages(numWatchers int, value string) {
psc := redigomock.NewConn() psc := redigomock.NewConn()
// Setup the initial subscription message // Setup the initial subscription message
@ -60,11 +54,11 @@ func processMessages(numWatchers int, value string) {
psc.AddSubscriptionMessage(createSubscriptionMessage(keySubChannel, runnerKey+"="+value)) psc.AddSubscriptionMessage(createSubscriptionMessage(keySubChannel, runnerKey+"="+value))
// Wait for all the `WatchKey` calls to be registered // Wait for all the `WatchKey` calls to be registered
for countWatchers(runnerKey) != numWatchers { for kw.countSubscribers(runnerKey) != numWatchers {
time.Sleep(time.Millisecond) time.Sleep(time.Millisecond)
} }
processInner(psc) kw.receivePubSubStream(psc)
} }
type keyChangeTestCase struct { type keyChangeTestCase struct {
@ -81,12 +75,13 @@ func TestKeyChangesBubblesUpError(t *testing.T) {
conn, td := setupMockPool() conn, td := setupMockPool()
defer td() defer td()
kw := NewKeyWatcher()
defer kw.Shutdown()
conn.Command("GET", runnerKey).ExpectError(errors.New("test error")) conn.Command("GET", runnerKey).ExpectError(errors.New("test error"))
_, err := WatchKey(runnerKey, "something", time.Second) _, err := kw.WatchKey(runnerKey, "something", time.Second)
require.Error(t, err, "Expected error") require.Error(t, err, "Expected error")
deleteWatchers(runnerKey)
} }
func TestKeyChangesInstantReturn(t *testing.T) { func TestKeyChangesInstantReturn(t *testing.T) {
@ -135,12 +130,13 @@ func TestKeyChangesInstantReturn(t *testing.T) {
conn.Command("GET", runnerKey).Expect(tc.returnValue) conn.Command("GET", runnerKey).Expect(tc.returnValue)
} }
val, err := WatchKey(runnerKey, tc.watchValue, tc.timeout) kw := NewKeyWatcher()
defer kw.Shutdown()
val, err := kw.WatchKey(runnerKey, tc.watchValue, tc.timeout)
require.NoError(t, err, "Expected no error") require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value") require.Equal(t, tc.expectedStatus, val, "Expected value")
deleteWatchers(runnerKey)
}) })
} }
} }
@ -183,18 +179,21 @@ func TestKeyChangesWhenWatching(t *testing.T) {
conn.Command("GET", runnerKey).Expect(tc.returnValue) conn.Command("GET", runnerKey).Expect(tc.returnValue)
} }
kw := NewKeyWatcher()
defer kw.Shutdown()
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
val, err := WatchKey(runnerKey, tc.watchValue, time.Second) val, err := kw.WatchKey(runnerKey, tc.watchValue, time.Second)
require.NoError(t, err, "Expected no error") require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value") require.Equal(t, tc.expectedStatus, val, "Expected value")
}() }()
processMessages(1, tc.processedValue) kw.processMessages(1, tc.processedValue)
wg.Wait() wg.Wait()
}) })
} }
@ -238,17 +237,20 @@ func TestKeyChangesParallel(t *testing.T) {
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
wg.Add(runTimes) wg.Add(runTimes)
kw := NewKeyWatcher()
defer kw.Shutdown()
for i := 0; i < runTimes; i++ { for i := 0; i < runTimes; i++ {
go func() { go func() {
defer wg.Done() defer wg.Done()
val, err := WatchKey(runnerKey, tc.watchValue, time.Second) val, err := kw.WatchKey(runnerKey, tc.watchValue, time.Second)
require.NoError(t, err, "Expected no error") require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value") require.Equal(t, tc.expectedStatus, val, "Expected value")
}() }()
} }
processMessages(runTimes, tc.processedValue) kw.processMessages(runTimes, tc.processedValue)
wg.Wait() wg.Wait()
}) })
} }
@ -257,7 +259,9 @@ func TestKeyChangesParallel(t *testing.T) {
func TestShutdown(t *testing.T) { func TestShutdown(t *testing.T) {
conn, td := setupMockPool() conn, td := setupMockPool()
defer td() defer td()
defer func() { shutdown = make(chan struct{}) }()
kw := NewKeyWatcher()
defer kw.Shutdown()
conn.Command("GET", runnerKey).Expect("something") conn.Command("GET", runnerKey).Expect("something")
@ -265,7 +269,7 @@ func TestShutdown(t *testing.T) {
wg.Add(2) wg.Add(2)
go func() { go func() {
val, err := WatchKey(runnerKey, "something", 10*time.Second) val, err := kw.WatchKey(runnerKey, "something", 10*time.Second)
require.NoError(t, err, "Expected no error") require.NoError(t, err, "Expected no error")
require.Equal(t, WatchKeyStatusNoChange, val, "Expected value not to change") require.Equal(t, WatchKeyStatusNoChange, val, "Expected value not to change")
@ -273,22 +277,22 @@ func TestShutdown(t *testing.T) {
}() }()
go func() { go func() {
require.Eventually(t, func() bool { return countWatchers(runnerKey) == 1 }, 10*time.Second, time.Millisecond) require.Eventually(t, func() bool { return kw.countSubscribers(runnerKey) == 1 }, 10*time.Second, time.Millisecond)
Shutdown() kw.Shutdown()
wg.Done() wg.Done()
}() }()
wg.Wait() wg.Wait()
require.Eventually(t, func() bool { return countWatchers(runnerKey) == 0 }, 10*time.Second, time.Millisecond) require.Eventually(t, func() bool { return kw.countSubscribers(runnerKey) == 0 }, 10*time.Second, time.Millisecond)
// Adding a key after the shutdown should result in an immediate response // Adding a key after the shutdown should result in an immediate response
var val WatchKeyStatus var val WatchKeyStatus
var err error var err error
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
val, err = WatchKey(runnerKey, "something", 10*time.Second) val, err = kw.WatchKey(runnerKey, "something", 10*time.Second)
close(done) close(done)
}() }()

2
workhorse/internal/upstream/metrics_test.go
View File

@ -26,7 +26,7 @@ func TestInstrumentGeoProxyRoute(t *testing.T) {
handleRouteWithMatchers(u, local), handleRouteWithMatchers(u, local),
handleRouteWithMatchers(u, main), handleRouteWithMatchers(u, main),
} }
}) }, nil)
ts := httptest.NewServer(u) ts := httptest.NewServer(u)
defer ts.Close() defer ts.Close()

3
workhorse/internal/upstream/routes.go
View File

@ -21,7 +21,6 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/imageresizer" "gitlab.com/gitlab-org/gitlab/workhorse/internal/imageresizer"
proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing" "gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/redis"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/secret" "gitlab.com/gitlab-org/gitlab/workhorse/internal/secret"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/senddata" "gitlab.com/gitlab-org/gitlab/workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/sendfile" "gitlab.com/gitlab-org/gitlab/workhorse/internal/sendfile"
@ -223,7 +222,7 @@ func configureRoutes(u *upstream) {
tempfileMultipartProxy := upload.FixedPreAuthMultipart(api, proxy, preparer) tempfileMultipartProxy := upload.FixedPreAuthMultipart(api, proxy, preparer)
ciAPIProxyQueue := queueing.QueueRequests("ci_api_job_requests", tempfileMultipartProxy, u.APILimit, u.APIQueueLimit, u.APIQueueTimeout) ciAPIProxyQueue := queueing.QueueRequests("ci_api_job_requests", tempfileMultipartProxy, u.APILimit, u.APIQueueLimit, u.APIQueueTimeout)
ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, redis.WatchKey, u.APICILongPollingDuration) ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, u.watchKeyHandler, u.APICILongPollingDuration)
dependencyProxyInjector.SetUploadHandler(requestBodyUploader) dependencyProxyInjector.SetUploadHandler(requestBodyUploader)

9
workhorse/internal/upstream/upstream.go
View File

@ -21,6 +21,7 @@ import (
"gitlab.com/gitlab-org/labkit/correlation" "gitlab.com/gitlab-org/labkit/correlation"
apipkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" apipkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/builds"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/config" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
@ -55,19 +56,21 @@ type upstream struct {
accessLogger *logrus.Logger accessLogger *logrus.Logger
enableGeoProxyFeature bool enableGeoProxyFeature bool
mu sync.RWMutex mu sync.RWMutex
watchKeyHandler builds.WatchKeyHandler
} }
func NewUpstream(cfg config.Config, accessLogger *logrus.Logger) http.Handler { func NewUpstream(cfg config.Config, accessLogger *logrus.Logger, watchKeyHandler builds.WatchKeyHandler) http.Handler {
return newUpstream(cfg, accessLogger, configureRoutes) return newUpstream(cfg, accessLogger, configureRoutes, watchKeyHandler)
} }
func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback func(*upstream)) http.Handler { func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback func(*upstream), watchKeyHandler builds.WatchKeyHandler) http.Handler {
up := upstream{ up := upstream{
Config: cfg, Config: cfg,
accessLogger: accessLogger, accessLogger: accessLogger,
// Kind of a feature flag. See https://gitlab.com/groups/gitlab-org/-/epics/5914#note_564974130 // Kind of a feature flag. See https://gitlab.com/groups/gitlab-org/-/epics/5914#note_564974130
enableGeoProxyFeature: os.Getenv("GEO_SECONDARY_PROXY") != "0", enableGeoProxyFeature: os.Getenv("GEO_SECONDARY_PROXY") != "0",
geoProxyBackend: &url.URL{}, geoProxyBackend: &url.URL{},
watchKeyHandler: watchKeyHandler,
} }
if up.geoProxyPollSleep == nil { if up.geoProxyPollSleep == nil {
up.geoProxyPollSleep = time.Sleep up.geoProxyPollSleep = time.Sleep

4
workhorse/internal/upstream/upstream_test.go
View File

@ -58,7 +58,7 @@ func TestRouting(t *testing.T) {
handle(u, quxbaz), handle(u, quxbaz),
handle(u, main), handle(u, main),
} }
}) }, nil)
ts := httptest.NewServer(u) ts := httptest.NewServer(u)
defer ts.Close() defer ts.Close()
@ -415,7 +415,7 @@ func startWorkhorseServer(railsServerURL string, enableGeoProxyFeature bool) (*h
configureRoutes(u) configureRoutes(u)
} }
cfg := newUpstreamConfig(railsServerURL) cfg := newUpstreamConfig(railsServerURL)
upstreamHandler := newUpstream(*cfg, logrus.StandardLogger(), myConfigureRoutes) upstreamHandler := newUpstream(*cfg, logrus.StandardLogger(), myConfigureRoutes, nil)
ws := httptest.NewServer(upstreamHandler) ws := httptest.NewServer(upstreamHandler)
waitForNextApiPoll := func() {} waitForNextApiPoll := func() {}

7
workhorse/main.go
View File

@ -220,9 +220,10 @@ func run(boot bootConfig, cfg config.Config) error {
secret.SetPath(boot.secretPath) secret.SetPath(boot.secretPath)
keyWatcher := redis.NewKeyWatcher()
if cfg.Redis != nil { if cfg.Redis != nil {
redis.Configure(cfg.Redis, redis.DefaultDialFunc) redis.Configure(cfg.Redis, redis.DefaultDialFunc)
go redis.Process() go keyWatcher.Process()
} }
if err := cfg.RegisterGoCloudURLOpeners(); err != nil { if err := cfg.RegisterGoCloudURLOpeners(); err != nil {
@ -237,7 +238,7 @@ func run(boot bootConfig, cfg config.Config) error {
gitaly.InitializeSidechannelRegistry(accessLogger) gitaly.InitializeSidechannelRegistry(accessLogger)
up := wrapRaven(upstream.NewUpstream(cfg, accessLogger)) up := wrapRaven(upstream.NewUpstream(cfg, accessLogger, keyWatcher.WatchKey))
done := make(chan os.Signal, 1) done := make(chan os.Signal, 1)
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
@ -271,7 +272,7 @@ func run(boot bootConfig, cfg config.Config) error {
ctx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout.Duration) // lint:allow context.Background ctx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout.Duration) // lint:allow context.Background
defer cancel() defer cancel()
redis.Shutdown() keyWatcher.Shutdown()
return srv.Shutdown(ctx) return srv.Shutdown(ctx)
} }
} }

2
workhorse/main_test.go
View File

@ -799,7 +799,7 @@ func startWorkhorseServer(authBackend string) *httptest.Server {
func startWorkhorseServerWithConfig(cfg *config.Config) *httptest.Server { func startWorkhorseServerWithConfig(cfg *config.Config) *httptest.Server {
testhelper.ConfigureSecret() testhelper.ConfigureSecret()
u := upstream.NewUpstream(*cfg, logrus.StandardLogger()) u := upstream.NewUpstream(*cfg, logrus.StandardLogger(), nil)
return httptest.NewServer(u) return httptest.NewServer(u)
} }

36
yarn.lock
View File

@ -1059,19 +1059,19 @@
stylelint-declaration-strict-value "1.8.0" stylelint-declaration-strict-value "1.8.0"
stylelint-scss "4.2.0" stylelint-scss "4.2.0"
"@gitlab/svgs@3.2.0": "@gitlab/svgs@3.3.0":
version "3.2.0" version "3.3.0"
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.2.0.tgz#1ff40355642600e8807775f2b137c184e46380e9" resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.3.0.tgz#99b044484fcf3d5a6431281e320e2405540ff5a9"
integrity sha512-djAEmvB3AljQaVKwEoNWls8Q6oWwGvUVrmtBe3ykyPF/E50QVmiM2kXIko2BAEPzmIKhaH9YchowfYqJX3y2vg== integrity sha512-S8Hqf+ms8aNrSgmci9SVoIyj/0qQnizU5uV5vUPAOwiufMDFDyI5qfcgn4EYZ6mnju3LiO+ReSL/PPTD4qNgHA==
"@gitlab/ui@43.9.1": "@gitlab/ui@43.9.3":
version "43.9.1" version "43.9.3"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-43.9.1.tgz#8864687ebaffe3ff71b8d6087ce55e3e52e57a79" resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-43.9.3.tgz#2dd91b14da769a873e45ffe07b5863f6c47211ba"
integrity sha512-1Rn4ZEOyQ0flDsAbxsFSnHNFqO0I2kuJjdkXfiEI21g0pdZ3LrdNwE0WcoRZWQd+nQQ0XbvzRaqmxN53rTY21g== integrity sha512-TONSf+6UJYWTVs5qnItR1uLZ/0kBE8jGN8aLOVv4CDAsORvln0ZxtcZvMTCFp76YEtzXLkMUfvm7ZngQ26tIiA==
dependencies: dependencies:
"@popperjs/core" "^2.11.2" "@popperjs/core" "^2.11.2"
bootstrap-vue "2.20.1" bootstrap-vue "2.20.1"
dompurify "^2.3.10" dompurify "^2.4.0"
echarts "^5.3.2" echarts "^5.3.2"
iframe-resizer "^4.3.2" iframe-resizer "^4.3.2"
lodash "^4.17.20" lodash "^4.17.20"
@ -4834,7 +4834,7 @@ dompurify@2.3.8:
resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f" resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f"
integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw== integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw==
dompurify@^2.3.10, dompurify@^2.4.0: dompurify@^2.4.0:
version "2.4.0" version "2.4.0"
resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.0.tgz#c9c88390f024c2823332615c9e20a453cf3825dd" resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.0.tgz#c9c88390f024c2823332615c9e20a453cf3825dd"
integrity sha512-Be9tbQMZds4a3C6xTmz68NlMfeONA//4dOavl/1rNw50E+/QO0KVpbcU0PcaW0nsQxurXls9ZocqFxk8R2mWEA== integrity sha512-Be9tbQMZds4a3C6xTmz68NlMfeONA//4dOavl/1rNw50E+/QO0KVpbcU0PcaW0nsQxurXls9ZocqFxk8R2mWEA==
@ -12119,10 +12119,10 @@ webpack-dev-middleware@^5.3.1:
range-parser "^1.2.1" range-parser "^1.2.1"
schema-utils "^4.0.0" schema-utils "^4.0.0"
webpack-dev-server@4.10.0: webpack-dev-server@4.10.1:
version "4.10.0" version "4.10.1"
resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.10.0.tgz#de270d0009eba050546912be90116e7fd740a9ca" resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.10.1.tgz#124ac9ac261e75303d74d95ab6712b4aec3e12ed"
integrity sha512-7dezwAs+k6yXVFZ+MaL8VnE+APobiO3zvpp3rBHe/HmWQ+avwh0Q3d0xxacOiBybZZ3syTZw9HXzpa3YNbAZDQ== integrity sha512-FIzMq3jbBarz3ld9l7rbM7m6Rj1lOsgq/DyLGMX/fPEB1UBUPtf5iL/4eNfhx8YYJTRlzfv107UfWSWcBK5Odw==
dependencies: dependencies:
"@types/bonjour" "^3.5.9" "@types/bonjour" "^3.5.9"
"@types/connect-history-api-fallback" "^1.3.5" "@types/connect-history-api-fallback" "^1.3.5"
@ -12442,10 +12442,10 @@ yarn-check-webpack-plugin@^1.2.0:
dependencies: dependencies:
chalk "^2.4.2" chalk "^2.4.2"
yarn-deduplicate@^5.0.2: yarn-deduplicate@^6.0.0:
version "5.0.2" version "6.0.0"
resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-5.0.2.tgz#b56484c94d8f1163a828bf20516607f89c078675" resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-6.0.0.tgz#91bc0b7b374efe24796606df2c6b00eabb5aab62"
integrity sha512-pxKa+dM7DMQ4X2vYLKqGCUgtEoTtdMVk9gNoIsxsMSP0rOV51IWFcKHfRIcZjAPNgHTrxz46sKB4xr7Nte7jdw== integrity sha512-HjGVvuy10hetOuXeexXXT77V+6FfgS+NiW3FsmQD88yfF2kBqTpChvMglyKUlQ0xXEcI77VJazll5qKKBl3ssw==
dependencies: dependencies:
"@yarnpkg/lockfile" "^1.1.0" "@yarnpkg/lockfile" "^1.1.0"
commander "^9.4.0" commander "^9.4.0"