Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-08-31 06:12:36 +00:00
parent a5bd90f43b
commit 136651d7cb
24 changed files with 225 additions and 148 deletions

View File

@ -12,6 +12,8 @@ class ProjectsController < Projects::ApplicationController
include SourcegraphDecorator
include PlanningHierarchy
REFS_LIMIT = 100
prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:rss) }
around_action :allow_gitaly_ref_name_caching, only: [:index, :show]
@ -309,6 +311,8 @@ class ProjectsController < Projects::ApplicationController
find_tags = true
find_commits = true
use_gitaly_pagination = Feature.enabled?(:use_gitaly_pagination_for_refs, @project)
unless find_refs.nil?
find_branches = find_refs.include?('branches')
find_tags = find_refs.include?('tags')
@ -318,13 +322,21 @@ class ProjectsController < Projects::ApplicationController
options = {}
if find_branches
branches = BranchesFinder.new(@repository, refs_params).execute.take(100).map(&:name)
branches = BranchesFinder.new(@repository, refs_params.merge(per_page: REFS_LIMIT))
.execute(gitaly_pagination: use_gitaly_pagination)
.take(REFS_LIMIT)
.map(&:name)
options['Branches'] = branches
end
if find_tags && @repository.tag_count.nonzero?
tags = TagsFinder.new(@repository, refs_params).execute
options['Tags'] = tags.take(100).map(&:name)
tags = TagsFinder.new(@repository, refs_params.merge(per_page: REFS_LIMIT))
.execute(gitaly_pagination: use_gitaly_pagination)
.take(REFS_LIMIT)
.map(&:name)
options['Tags'] = tags
end
# If reference is commit id - we should add it to branch/tag selectbox

View File

@ -0,0 +1,8 @@
---
name: use_gitaly_pagination_for_refs
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96448
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/372049
milestone: '15.4'
type: development
group: group::source code
default_enabled: false

View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class DropTmpIndexTodosAttentionRequestActionIdx < Gitlab::Database::Migration[2.0]
INDEX_NAME = "tmp_index_todos_attention_request_action"
ATTENTION_REQUESTED = 10
disable_ddl_transaction!
def up
remove_concurrent_index_by_name :todos, INDEX_NAME
end
def down
add_concurrent_index :todos, [:id],
where: "action = #{ATTENTION_REQUESTED}",
name: INDEX_NAME
end
end

View File

@ -0,0 +1 @@
0338843ad56b423559e613f00df205122b4f6db194cf49712b2ff46b2ad030e0

View File

@ -30668,8 +30668,6 @@ CREATE INDEX tmp_index_project_statistics_cont_registry_size ON project_statisti
CREATE INDEX tmp_index_system_note_metadata_on_id_where_task ON system_note_metadata USING btree (id, action) WHERE ((action)::text = 'task'::text);
CREATE INDEX tmp_index_todos_attention_request_action ON todos USING btree (id) WHERE (action = 10);
CREATE INDEX tmp_index_vulnerability_occurrences_on_id_and_scanner_id ON vulnerability_occurrences USING btree (id, scanner_id) WHERE (report_type = ANY (ARRAY[7, 99]));
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);

View File

@ -82,7 +82,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
#### Customizing GitLab appearance
- [Header logo](../user/admin_area/appearance.md#navigation-bar): Change the logo on all pages and email headers.
- [Header logo](../user/admin_area/appearance.md#top-bar): Change the logo on all pages and email headers.
- [Favicon](../user/admin_area/appearance.md#favicon): Change the default favicon to your own logo.
- [Branded login page](../user/admin_area/appearance.md#sign-in--sign-up-pages): Customize the login page with your own logo, title, and description.
- ["New Project" page](../user/admin_area/appearance.md#new-project-pages): Customize the text to be displayed on the page that opens whenever your users create a new project.

View File

@ -13,9 +13,9 @@ of GitLab. To access these settings:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Appearance**.
## Navigation bar
## Top bar
By default, the navigation bar has the GitLab logo, but this can be customized with
By default, the **top bar** has the GitLab logo, but this can be customized with
any image desired. It is optimized for images 28px high (any width), but any image can be
used (less than 1 MB) and it is automatically resized.

View File

@ -11,7 +11,7 @@ You can customize some of the content in emails sent from your GitLab instance.
## Custom logo
The logo in the header of some emails can be customized, see the [logo customization section](../appearance.md#navigation-bar).
The logo in the header of some emails can be customized, see the [logo customization section](../appearance.md#top-bar).
## Include author name in email notification email body **(PREMIUM SELF)**

View File

@ -52,8 +52,8 @@
"@codesandbox/sandpack-client": "^1.2.2",
"@gitlab/at.js": "1.5.7",
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/svgs": "3.2.0",
"@gitlab/ui": "43.9.1",
"@gitlab/svgs": "3.3.0",
"@gitlab/ui": "43.9.3",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20220815034418",
"@rails/actioncable": "6.1.4-7",
@ -244,10 +244,10 @@
"sass": "^1.49.9",
"stylelint": "^14.9.1",
"timezone-mock": "^1.0.8",
"webpack-dev-server": "4.10.0",
"webpack-dev-server": "4.10.1",
"xhr-mock": "^2.5.1",
"yarn-check-webpack-plugin": "^1.2.0",
"yarn-deduplicate": "^5.0.2"
"yarn-deduplicate": "^6.0.0"
},
"blockedDependencies": {
"bootstrap-vue": "https://docs.gitlab.com/ee/development/fe_guide/dependencies.html#bootstrapvue"

View File

@ -88,14 +88,19 @@ module QA
let(:gh_issue_comments) do
logger.debug("= Fetching issue comments =")
github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url) # use base html url as key
# use base html url as key
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url)
end
end
let(:gh_pr_comments) do
logger.debug("= Fetching pr comments =")
github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash|
hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url) # use base html url as key
# use base html url as key
hash[c.html_url.gsub(/\#\S+/, "")] << c.body
# some suggestions can contain extra whitespaces which gitlab will remove
&.gsub(/suggestion\s+\r/, "suggestion\r")
&.gsub(gh_link_pattern, dummy_url)
end
end

View File

@ -103,7 +103,7 @@ module QA
expect(response.headers[:expires]).to eq("Fri, 01 Jan 1990 00:00:00 GMT")
expect(response.headers[:content_disposition]).to include("attachment")
expect(response.headers[:content_disposition]).not_to include("inline")
expect(response.headers[:content_type]).to include("application/octet-stream")
expect(response.headers[:content_type]).to include("image/svg+xml")
end
delete_project_request = Runtime::API::Request.new(@api_client, "/projects/#{sanitized_project_path}")

View File

@ -20,8 +20,10 @@ module QA
Flow::Login.sign_in(as: user)
Page::Dashboard::Welcome.perform do |welcome|
expect(welcome).to have_welcome_title("Welcome to GitLab")
Support::Waiter.wait_until(sleep_interval: 2, max_duration: 60, reload_page: page,
retry_on_exception: true) do
expect(welcome).to have_welcome_title("Welcome to GitLab")
end
# This would be better if it were a visual validation test
expect(welcome).to have_loaded_all_images
end

View File

@ -1217,6 +1217,40 @@ RSpec.describe ProjectsController do
expect(json_response["Commits"]).to include("123456")
end
it 'uses gitaly pagination' do
expected_params = ActionController::Parameters.new(ref: '123456', per_page: 100).permit!
expect_next_instance_of(BranchesFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: true).and_call_original
end
expect_next_instance_of(TagsFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: true).and_call_original
end
get :refs, params: { namespace_id: project.namespace, id: project, ref: "123456" }
end
context 'when use_gitaly_pagination_for_refs is disabled' do
before do
stub_feature_flags(use_gitaly_pagination_for_refs: false)
end
it 'does not use gitaly pagination' do
expected_params = ActionController::Parameters.new(ref: '123456', per_page: 100).permit!
expect_next_instance_of(BranchesFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: false).and_call_original
end
expect_next_instance_of(TagsFinder, project.repository, expected_params) do |finder|
expect(finder).to receive(:execute).with(gitaly_pagination: false).and_call_original
end
get :refs, params: { namespace_id: project.namespace, id: project, ref: "123456" }
end
end
context 'when gitaly is unavailable' do
before do
expect_next_instance_of(TagsFinder) do |finder|

View File

@ -26,7 +26,7 @@ require (
github.com/sirupsen/logrus v1.9.0
github.com/smartystreets/goconvey v1.7.2
github.com/stretchr/testify v1.8.0
gitlab.com/gitlab-org/gitaly/v15 v15.2.2
gitlab.com/gitlab-org/gitaly/v15 v15.3.2
gitlab.com/gitlab-org/golang-archive-zip v0.1.1
gitlab.com/gitlab-org/labkit v1.16.0
gocloud.dev v0.25.0
@ -74,7 +74,7 @@ require (
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/kr/text v0.2.0 // indirect

View File

@ -639,8 +639,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I=
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
github.com/hhatto/gorst v0.0.0-20181029133204-ca9f730cac5b/go.mod h1:HmaZGXHdSwQh1jnUlBGN2BeEYOHACLVGzYOXCbsLvxY=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -1125,11 +1125,11 @@ gitlab.com/gitlab-org/gitaly v1.68.0 h1:VlcJs1+PrhW7lqJUU7Fh1q8FMJujmbbivdfde/cw
gitlab.com/gitlab-org/gitaly v1.68.0/go.mod h1:/pCsB918Zu5wFchZ9hLYin9WkJ2yQqdVNz0zlv5HbXg=
gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1/go.mod h1:4Cz8tOAyueSZX5o6gYum1F/unupaOclxqETPcg4ODvQ=
gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc5.0.20220329111719-51da8bc17059/go.mod h1:uX1qhFKBDuPqATlpMcFL2dKDiX8D/tbUg7CYWx7OXt4=
gitlab.com/gitlab-org/gitaly/v15 v15.2.2 h1:/hSbAhBqRrT6Epc35k83qFwwVbKottNY6wDFr+5DYQo=
gitlab.com/gitlab-org/gitaly/v15 v15.2.2/go.mod h1:WjitFL44l9ovitGC4OvSuGwfeq0VpHUbHS6sDw13LV8=
gitlab.com/gitlab-org/gitaly/v15 v15.3.2 h1:H8NoBZil23mZ8Y31XdPWCNp27m2nfFGV+z/pQd5Rsq4=
gitlab.com/gitlab-org/gitaly/v15 v15.3.2/go.mod h1:DPO/d7DnhJ0TTcL6UZNJ6mcRHhdBPg+f/iBA+LEKEq0=
gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h1:5QSTbpAHY2v0iIH5uHh2KA9w7sPUqPmnLjDApI/sv1U=
gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20210720163109-50da611814d2/go.mod h1:QWDYBwuy24qGMandtCngLRPzFgnGPg6LSNoJWPKmJMc=
gitlab.com/gitlab-org/gitlab-shell/v14 v14.8.0/go.mod h1:Z1S5MihpEmtA7GDXGsU0kUf1nzm7zr8w6bP+uXRnxaw=
gitlab.com/gitlab-org/gitlab-shell/v14 v14.10.0/go.mod h1:ynuwa2oLLQSOs6Ss6RpLDkAQuFUNreI3NdEOJxfh1ac=
gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE=
gitlab.com/gitlab-org/golang-archive-zip v0.1.1/go.mod h1:ZDtqpWPGPB9qBuZnZDrKQjIdJtkN7ZAoVwhT6H2o2kE=
gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM=
@ -1137,8 +1137,6 @@ gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029/go.mod h1:SNfxkf
gitlab.com/gitlab-org/labkit v1.0.0/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U=
gitlab.com/gitlab-org/labkit v1.4.1/go.mod h1:x5JO5uvdX4t6e/TZXLXZnFL5AcKz2uLLd3uKXZcuO4k=
gitlab.com/gitlab-org/labkit v1.5.0/go.mod h1:1ZuVZpjSpCKUgjLx8P6jzkkQFxJI1thUKr6yKV3p0vY=
gitlab.com/gitlab-org/labkit v1.14.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
gitlab.com/gitlab-org/labkit v1.15.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
gitlab.com/gitlab-org/labkit v1.16.0 h1:Vm3NAMZ8RqAunXlvPWby3GJ2R35vsYGP6Uu0YjyMIlY=
gitlab.com/gitlab-org/labkit v1.16.0/go.mod h1:bcxc4ZpAC+WyACgyKl7FcvT2XXAbl8CrzN6UY+w8cMc=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@ -1477,7 +1475,6 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@ -1782,7 +1779,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=

View File

@ -15,17 +15,27 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/log"
)
var (
keyWatcher = make(map[string][]chan string)
keyWatcherMutex sync.Mutex
shutdown = make(chan struct{})
redisReconnectTimeout = backoff.Backoff{
//These are the defaults
Min: 100 * time.Millisecond,
Max: 60 * time.Second,
Factor: 2,
Jitter: true,
type KeyWatcher struct {
mu sync.Mutex
subscribers map[string][]chan string
shutdown chan struct{}
reconnectBackoff backoff.Backoff
}
func NewKeyWatcher() *KeyWatcher {
return &KeyWatcher{
subscribers: make(map[string][]chan string),
shutdown: make(chan struct{}),
reconnectBackoff: backoff.Backoff{
Min: 100 * time.Millisecond,
Max: 60 * time.Second,
Factor: 2,
Jitter: true,
},
}
}
var (
keyWatchers = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "gitlab_workhorse_keywatcher_keywatchers",
@ -57,15 +67,9 @@ const (
keySubChannel = "workhorse:notifications"
)
// KeyChan holds a key and a channel
type KeyChan struct {
Key string
Chan chan string
}
func countAction(action string) { totalActions.WithLabelValues(action).Add(1) }
func processInner(conn redis.Conn) error {
func (kw *KeyWatcher) receivePubSubStream(conn redis.Conn) error {
defer conn.Close()
psc := redis.PubSubConn{Conn: conn}
if err := psc.Subscribe(keySubChannel); err != nil {
@ -84,7 +88,7 @@ func processInner(conn redis.Conn) error {
log.WithError(fmt.Errorf("keywatcher: invalid notification: %q", dataStr)).Error()
continue
}
notifyChanWatchers(msg[0], msg[1])
kw.notifySubscribers(msg[0], msg[1])
case error:
log.WithError(fmt.Errorf("keywatcher: pubsub receive: %v", v)).Error()
// Intermittent error, return nil so that it doesn't wait before reconnect
@ -109,45 +113,42 @@ func dialPubSub(dialer redisDialerFunc) (redis.Conn, error) {
return conn, nil
}
// Process redis subscriptions
//
// NOTE: There Can Only Be One!
func Process() {
func (kw *KeyWatcher) Process() {
log.Info("keywatcher: starting process loop")
for {
conn, err := dialPubSub(workerDialFunc)
if err != nil {
log.WithError(fmt.Errorf("keywatcher: %v", err)).Error()
time.Sleep(redisReconnectTimeout.Duration())
time.Sleep(kw.reconnectBackoff.Duration())
continue
}
redisReconnectTimeout.Reset()
kw.reconnectBackoff.Reset()
if err = processInner(conn); err != nil {
log.WithError(fmt.Errorf("keywatcher: process loop: %v", err)).Error()
if err = kw.receivePubSubStream(conn); err != nil {
log.WithError(fmt.Errorf("keywatcher: receivePubSubStream: %v", err)).Error()
}
}
}
func Shutdown() {
func (kw *KeyWatcher) Shutdown() {
log.Info("keywatcher: shutting down")
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
kw.mu.Lock()
defer kw.mu.Unlock()
select {
case <-shutdown:
case <-kw.shutdown:
// already closed
default:
close(shutdown)
close(kw.shutdown)
}
}
func notifyChanWatchers(key, value string) {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
func (kw *KeyWatcher) notifySubscribers(key, value string) {
kw.mu.Lock()
defer kw.mu.Unlock()
chanList, ok := keyWatcher[key]
chanList, ok := kw.subscribers[key]
if !ok {
countAction("drop-message")
return
@ -158,38 +159,38 @@ func notifyChanWatchers(key, value string) {
c <- value
keyWatchers.Dec()
}
delete(keyWatcher, key)
delete(kw.subscribers, key)
}
func addKeyChan(kc *KeyChan) {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
func (kw *KeyWatcher) addSubscription(key string, notify chan string) {
kw.mu.Lock()
defer kw.mu.Unlock()
keyWatcher[kc.Key] = append(keyWatcher[kc.Key], kc.Chan)
kw.subscribers[key] = append(kw.subscribers[key], notify)
keyWatchers.Inc()
if len(keyWatcher[kc.Key]) == 1 {
if len(kw.subscribers[key]) == 1 {
countAction("create-subscription")
}
}
func delKeyChan(kc *KeyChan) {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
func (kw *KeyWatcher) delSubscription(key string, notify chan string) {
kw.mu.Lock()
defer kw.mu.Unlock()
chans, ok := keyWatcher[kc.Key]
chans, ok := kw.subscribers[key]
if !ok {
return
}
for i, c := range chans {
if kc.Chan == c {
keyWatcher[kc.Key] = append(chans[:i], chans[i+1:]...)
if notify == c {
kw.subscribers[key] = append(chans[:i], chans[i+1:]...)
keyWatchers.Dec()
break
}
}
if len(keyWatcher[kc.Key]) == 0 {
delete(keyWatcher, kc.Key)
if len(kw.subscribers[key]) == 0 {
delete(kw.subscribers, key)
countAction("delete-subscription")
}
}
@ -209,15 +210,10 @@ const (
WatchKeyStatusNoChange
)
// WatchKey waits for a key to be updated or expired
func WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error) {
kw := &KeyChan{
Key: key,
Chan: make(chan string, 1),
}
addKeyChan(kw)
defer delKeyChan(kw)
func (kw *KeyWatcher) WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error) {
notify := make(chan string, 1)
kw.addSubscription(key, notify)
defer kw.delSubscription(key, notify)
currentValue, err := GetString(key)
if errors.Is(err, redis.ErrNil) {
@ -230,10 +226,10 @@ func WatchKey(key, value string, timeout time.Duration) (WatchKeyStatus, error)
}
select {
case <-shutdown:
case <-kw.shutdown:
log.WithFields(log.Fields{"key": key}).Info("stopping watch due to shutdown")
return WatchKeyStatusNoChange, nil
case currentValue := <-kw.Chan:
case currentValue := <-notify:
if currentValue == "" {
return WatchKeyStatusNoChange, fmt.Errorf("keywatcher: redis GET failed")
}

View File

@ -38,20 +38,14 @@ func createUnsubscribeMessage(key string) []interface{} {
}
}
func countWatchers(key string) int {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
return len(keyWatcher[key])
}
func deleteWatchers(key string) {
keyWatcherMutex.Lock()
defer keyWatcherMutex.Unlock()
delete(keyWatcher, key)
func (kw *KeyWatcher) countSubscribers(key string) int {
kw.mu.Lock()
defer kw.mu.Unlock()
return len(kw.subscribers[key])
}
// Forces a run of the `Process` loop against a mock PubSubConn.
func processMessages(numWatchers int, value string) {
func (kw *KeyWatcher) processMessages(numWatchers int, value string) {
psc := redigomock.NewConn()
// Setup the initial subscription message
@ -60,11 +54,11 @@ func processMessages(numWatchers int, value string) {
psc.AddSubscriptionMessage(createSubscriptionMessage(keySubChannel, runnerKey+"="+value))
// Wait for all the `WatchKey` calls to be registered
for countWatchers(runnerKey) != numWatchers {
for kw.countSubscribers(runnerKey) != numWatchers {
time.Sleep(time.Millisecond)
}
processInner(psc)
kw.receivePubSubStream(psc)
}
type keyChangeTestCase struct {
@ -81,12 +75,13 @@ func TestKeyChangesBubblesUpError(t *testing.T) {
conn, td := setupMockPool()
defer td()
kw := NewKeyWatcher()
defer kw.Shutdown()
conn.Command("GET", runnerKey).ExpectError(errors.New("test error"))
_, err := WatchKey(runnerKey, "something", time.Second)
_, err := kw.WatchKey(runnerKey, "something", time.Second)
require.Error(t, err, "Expected error")
deleteWatchers(runnerKey)
}
func TestKeyChangesInstantReturn(t *testing.T) {
@ -135,12 +130,13 @@ func TestKeyChangesInstantReturn(t *testing.T) {
conn.Command("GET", runnerKey).Expect(tc.returnValue)
}
val, err := WatchKey(runnerKey, tc.watchValue, tc.timeout)
kw := NewKeyWatcher()
defer kw.Shutdown()
val, err := kw.WatchKey(runnerKey, tc.watchValue, tc.timeout)
require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value")
deleteWatchers(runnerKey)
})
}
}
@ -183,18 +179,21 @@ func TestKeyChangesWhenWatching(t *testing.T) {
conn.Command("GET", runnerKey).Expect(tc.returnValue)
}
kw := NewKeyWatcher()
defer kw.Shutdown()
wg := &sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
val, err := WatchKey(runnerKey, tc.watchValue, time.Second)
val, err := kw.WatchKey(runnerKey, tc.watchValue, time.Second)
require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value")
}()
processMessages(1, tc.processedValue)
kw.processMessages(1, tc.processedValue)
wg.Wait()
})
}
@ -238,17 +237,20 @@ func TestKeyChangesParallel(t *testing.T) {
wg := &sync.WaitGroup{}
wg.Add(runTimes)
kw := NewKeyWatcher()
defer kw.Shutdown()
for i := 0; i < runTimes; i++ {
go func() {
defer wg.Done()
val, err := WatchKey(runnerKey, tc.watchValue, time.Second)
val, err := kw.WatchKey(runnerKey, tc.watchValue, time.Second)
require.NoError(t, err, "Expected no error")
require.Equal(t, tc.expectedStatus, val, "Expected value")
}()
}
processMessages(runTimes, tc.processedValue)
kw.processMessages(runTimes, tc.processedValue)
wg.Wait()
})
}
@ -257,7 +259,9 @@ func TestKeyChangesParallel(t *testing.T) {
func TestShutdown(t *testing.T) {
conn, td := setupMockPool()
defer td()
defer func() { shutdown = make(chan struct{}) }()
kw := NewKeyWatcher()
defer kw.Shutdown()
conn.Command("GET", runnerKey).Expect("something")
@ -265,7 +269,7 @@ func TestShutdown(t *testing.T) {
wg.Add(2)
go func() {
val, err := WatchKey(runnerKey, "something", 10*time.Second)
val, err := kw.WatchKey(runnerKey, "something", 10*time.Second)
require.NoError(t, err, "Expected no error")
require.Equal(t, WatchKeyStatusNoChange, val, "Expected value not to change")
@ -273,22 +277,22 @@ func TestShutdown(t *testing.T) {
}()
go func() {
require.Eventually(t, func() bool { return countWatchers(runnerKey) == 1 }, 10*time.Second, time.Millisecond)
require.Eventually(t, func() bool { return kw.countSubscribers(runnerKey) == 1 }, 10*time.Second, time.Millisecond)
Shutdown()
kw.Shutdown()
wg.Done()
}()
wg.Wait()
require.Eventually(t, func() bool { return countWatchers(runnerKey) == 0 }, 10*time.Second, time.Millisecond)
require.Eventually(t, func() bool { return kw.countSubscribers(runnerKey) == 0 }, 10*time.Second, time.Millisecond)
// Adding a key after the shutdown should result in an immediate response
var val WatchKeyStatus
var err error
done := make(chan struct{})
go func() {
val, err = WatchKey(runnerKey, "something", 10*time.Second)
val, err = kw.WatchKey(runnerKey, "something", 10*time.Second)
close(done)
}()

View File

@ -26,7 +26,7 @@ func TestInstrumentGeoProxyRoute(t *testing.T) {
handleRouteWithMatchers(u, local),
handleRouteWithMatchers(u, main),
}
})
}, nil)
ts := httptest.NewServer(u)
defer ts.Close()

View File

@ -21,7 +21,6 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/imageresizer"
proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/redis"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/secret"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/sendfile"
@ -223,7 +222,7 @@ func configureRoutes(u *upstream) {
tempfileMultipartProxy := upload.FixedPreAuthMultipart(api, proxy, preparer)
ciAPIProxyQueue := queueing.QueueRequests("ci_api_job_requests", tempfileMultipartProxy, u.APILimit, u.APIQueueLimit, u.APIQueueTimeout)
ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, redis.WatchKey, u.APICILongPollingDuration)
ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, u.watchKeyHandler, u.APICILongPollingDuration)
dependencyProxyInjector.SetUploadHandler(requestBodyUploader)

View File

@ -21,6 +21,7 @@ import (
"gitlab.com/gitlab-org/labkit/correlation"
apipkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/builds"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
@ -55,19 +56,21 @@ type upstream struct {
accessLogger *logrus.Logger
enableGeoProxyFeature bool
mu sync.RWMutex
watchKeyHandler builds.WatchKeyHandler
}
func NewUpstream(cfg config.Config, accessLogger *logrus.Logger) http.Handler {
return newUpstream(cfg, accessLogger, configureRoutes)
func NewUpstream(cfg config.Config, accessLogger *logrus.Logger, watchKeyHandler builds.WatchKeyHandler) http.Handler {
return newUpstream(cfg, accessLogger, configureRoutes, watchKeyHandler)
}
func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback func(*upstream)) http.Handler {
func newUpstream(cfg config.Config, accessLogger *logrus.Logger, routesCallback func(*upstream), watchKeyHandler builds.WatchKeyHandler) http.Handler {
up := upstream{
Config: cfg,
accessLogger: accessLogger,
// Kind of a feature flag. See https://gitlab.com/groups/gitlab-org/-/epics/5914#note_564974130
enableGeoProxyFeature: os.Getenv("GEO_SECONDARY_PROXY") != "0",
geoProxyBackend: &url.URL{},
watchKeyHandler: watchKeyHandler,
}
if up.geoProxyPollSleep == nil {
up.geoProxyPollSleep = time.Sleep

View File

@ -58,7 +58,7 @@ func TestRouting(t *testing.T) {
handle(u, quxbaz),
handle(u, main),
}
})
}, nil)
ts := httptest.NewServer(u)
defer ts.Close()
@ -415,7 +415,7 @@ func startWorkhorseServer(railsServerURL string, enableGeoProxyFeature bool) (*h
configureRoutes(u)
}
cfg := newUpstreamConfig(railsServerURL)
upstreamHandler := newUpstream(*cfg, logrus.StandardLogger(), myConfigureRoutes)
upstreamHandler := newUpstream(*cfg, logrus.StandardLogger(), myConfigureRoutes, nil)
ws := httptest.NewServer(upstreamHandler)
waitForNextApiPoll := func() {}

View File

@ -220,9 +220,10 @@ func run(boot bootConfig, cfg config.Config) error {
secret.SetPath(boot.secretPath)
keyWatcher := redis.NewKeyWatcher()
if cfg.Redis != nil {
redis.Configure(cfg.Redis, redis.DefaultDialFunc)
go redis.Process()
go keyWatcher.Process()
}
if err := cfg.RegisterGoCloudURLOpeners(); err != nil {
@ -237,7 +238,7 @@ func run(boot bootConfig, cfg config.Config) error {
gitaly.InitializeSidechannelRegistry(accessLogger)
up := wrapRaven(upstream.NewUpstream(cfg, accessLogger))
up := wrapRaven(upstream.NewUpstream(cfg, accessLogger, keyWatcher.WatchKey))
done := make(chan os.Signal, 1)
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
@ -271,7 +272,7 @@ func run(boot bootConfig, cfg config.Config) error {
ctx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout.Duration) // lint:allow context.Background
defer cancel()
redis.Shutdown()
keyWatcher.Shutdown()
return srv.Shutdown(ctx)
}
}

View File

@ -799,7 +799,7 @@ func startWorkhorseServer(authBackend string) *httptest.Server {
func startWorkhorseServerWithConfig(cfg *config.Config) *httptest.Server {
testhelper.ConfigureSecret()
u := upstream.NewUpstream(*cfg, logrus.StandardLogger())
u := upstream.NewUpstream(*cfg, logrus.StandardLogger(), nil)
return httptest.NewServer(u)
}

View File

@ -1059,19 +1059,19 @@
stylelint-declaration-strict-value "1.8.0"
stylelint-scss "4.2.0"
"@gitlab/svgs@3.2.0":
version "3.2.0"
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.2.0.tgz#1ff40355642600e8807775f2b137c184e46380e9"
integrity sha512-djAEmvB3AljQaVKwEoNWls8Q6oWwGvUVrmtBe3ykyPF/E50QVmiM2kXIko2BAEPzmIKhaH9YchowfYqJX3y2vg==
"@gitlab/svgs@3.3.0":
version "3.3.0"
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.3.0.tgz#99b044484fcf3d5a6431281e320e2405540ff5a9"
integrity sha512-S8Hqf+ms8aNrSgmci9SVoIyj/0qQnizU5uV5vUPAOwiufMDFDyI5qfcgn4EYZ6mnju3LiO+ReSL/PPTD4qNgHA==
"@gitlab/ui@43.9.1":
version "43.9.1"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-43.9.1.tgz#8864687ebaffe3ff71b8d6087ce55e3e52e57a79"
integrity sha512-1Rn4ZEOyQ0flDsAbxsFSnHNFqO0I2kuJjdkXfiEI21g0pdZ3LrdNwE0WcoRZWQd+nQQ0XbvzRaqmxN53rTY21g==
"@gitlab/ui@43.9.3":
version "43.9.3"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-43.9.3.tgz#2dd91b14da769a873e45ffe07b5863f6c47211ba"
integrity sha512-TONSf+6UJYWTVs5qnItR1uLZ/0kBE8jGN8aLOVv4CDAsORvln0ZxtcZvMTCFp76YEtzXLkMUfvm7ZngQ26tIiA==
dependencies:
"@popperjs/core" "^2.11.2"
bootstrap-vue "2.20.1"
dompurify "^2.3.10"
dompurify "^2.4.0"
echarts "^5.3.2"
iframe-resizer "^4.3.2"
lodash "^4.17.20"
@ -4834,7 +4834,7 @@ dompurify@2.3.8:
resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f"
integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw==
dompurify@^2.3.10, dompurify@^2.4.0:
dompurify@^2.4.0:
version "2.4.0"
resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.0.tgz#c9c88390f024c2823332615c9e20a453cf3825dd"
integrity sha512-Be9tbQMZds4a3C6xTmz68NlMfeONA//4dOavl/1rNw50E+/QO0KVpbcU0PcaW0nsQxurXls9ZocqFxk8R2mWEA==
@ -12119,10 +12119,10 @@ webpack-dev-middleware@^5.3.1:
range-parser "^1.2.1"
schema-utils "^4.0.0"
webpack-dev-server@4.10.0:
version "4.10.0"
resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.10.0.tgz#de270d0009eba050546912be90116e7fd740a9ca"
integrity sha512-7dezwAs+k6yXVFZ+MaL8VnE+APobiO3zvpp3rBHe/HmWQ+avwh0Q3d0xxacOiBybZZ3syTZw9HXzpa3YNbAZDQ==
webpack-dev-server@4.10.1:
version "4.10.1"
resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.10.1.tgz#124ac9ac261e75303d74d95ab6712b4aec3e12ed"
integrity sha512-FIzMq3jbBarz3ld9l7rbM7m6Rj1lOsgq/DyLGMX/fPEB1UBUPtf5iL/4eNfhx8YYJTRlzfv107UfWSWcBK5Odw==
dependencies:
"@types/bonjour" "^3.5.9"
"@types/connect-history-api-fallback" "^1.3.5"
@ -12442,10 +12442,10 @@ yarn-check-webpack-plugin@^1.2.0:
dependencies:
chalk "^2.4.2"
yarn-deduplicate@^5.0.2:
version "5.0.2"
resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-5.0.2.tgz#b56484c94d8f1163a828bf20516607f89c078675"
integrity sha512-pxKa+dM7DMQ4X2vYLKqGCUgtEoTtdMVk9gNoIsxsMSP0rOV51IWFcKHfRIcZjAPNgHTrxz46sKB4xr7Nte7jdw==
yarn-deduplicate@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-6.0.0.tgz#91bc0b7b374efe24796606df2c6b00eabb5aab62"
integrity sha512-HjGVvuy10hetOuXeexXXT77V+6FfgS+NiW3FsmQD88yfF2kBqTpChvMglyKUlQ0xXEcI77VJazll5qKKBl3ssw==
dependencies:
"@yarnpkg/lockfile" "^1.1.0"
commander "^9.4.0"