Add latest changes from gitlab-org/gitlab@master

app/controllers/jira_connect/application_controller.rb

@@ -22,6 +22,8 @@ class JiraConnect::ApplicationController < ApplicationController
   def verify_qsh_claim!
     payload, _ = decode_auth_token!
 
+    return if request.format.json? && payload['qsh'] == 'context-qsh'
+
     # Make sure `qsh` claim matches the current request
     render_403 unless payload['qsh'] == Atlassian::Jwt.create_query_string_hash(request.url, request.method, jira_connect_base_url)
   rescue StandardError

app/views/admin/application_settings/_grafana.html.haml

@@ -1,14 +1,11 @@
-= form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
+= gitlab_ui_form_for @application_setting, url: metrics_and_profiling_admin_application_settings_path(anchor: 'js-grafana-settings'), html: { class: 'fieldset-form' } do |f|
   = form_errors(@application_setting)
 
   %fieldset
     .form-group
-      .form-check
-        = f.check_box :grafana_enabled, class: 'form-check-input'
-        = f.label :grafana_enabled, class: 'form-check-label' do
-          = _("Add a link to Grafana")
-        .form-text.text-muted
-          = _("A Metrics Dashboard menu item appears in the Monitoring section of the Admin Area.")
+      = f.gitlab_ui_checkbox_component :grafana_enabled,
+      s_('ApplicationSettings|Add a link to Grafana'),
+      help_text: s_('ApplicationSettings|A Metrics Dashboard menu item appears in the Monitoring section of the Admin Area.')
     .form-group
       = f.label :grafana_url, _('Grafana URL'), class: 'label-bold'
       = f.text_field :grafana_url, class: 'form-control gl-form-input', placeholder: '/-/grafana'

app/views/devise/sessions/_new_base.html.haml

@@ -1,4 +1,4 @@
-= form_for(resource, as: resource_name, url: session_path(resource_name), html: { class: 'new_user gl-show-field-errors js-sign-in-form', aria: { live: 'assertive' }, data: { testid: 'sign-in-form' }}) do |f|
+= gitlab_ui_form_for(resource, as: resource_name, url: session_path(resource_name), html: { class: 'new_user gl-show-field-errors js-sign-in-form', aria: { live: 'assertive' }, data: { testid: 'sign-in-form' }}) do |f|
   .form-group
     = f.label _('Username or email'), for: 'user_login', class: 'label-bold'
     = f.text_field :login, value: @invite_email, class: 'form-control gl-form-input top js-username-field', autofocus: 'autofocus', autocapitalize: 'off', autocorrect: 'off', required: true, title: _('This field is required.'), data: { qa_selector: 'login_field', testid: 'username-field' }
@@ -7,10 +7,9 @@
     = f.password_field :password, class: 'form-control gl-form-input bottom', autocomplete: 'current-password', required: true, title: _('This field is required.'), data: { qa_selector: 'password_field' }
   - if devise_mapping.rememberable?
     %div
-      %label{ for: 'user_remember_me' }
-        = f.check_box :remember_me
-        %span= _('Remember me')
-      .float-right
+      .gl-display-inline-block
+        = f.gitlab_ui_checkbox_component :remember_me, _('Remember me')
+      .gl-float-right
         - if unconfirmed_email?
           = link_to _('Resend confirmation email'), new_user_confirmation_path
         - else

doc/user/group/saml_sso/index.md

@@ -23,12 +23,14 @@ If required, you can find [a glossary of common terms](../../../integration/saml
 
 ## Configure your identity provider
 
-1. On the top bar, select **Menu > Groups** and find your group.
-1. On the left sidebar, select **Settings > SAML SSO**.
-1. Configure your SAML identity provider using the **Assertion consumer service URL**, **Identifier**, and **GitLab single sign-on URL**.
-   Alternatively GitLab provides [metadata XML configuration](#metadata-configuration).
+1. Find the information in GitLab required for configuration:
+   1. On the top bar, select **Menu > Groups** and find your group.
+   1. On the left sidebar, select **Settings > SAML SSO**.
+   1. Note the **Assertion consumer service URL**, **Identifier**, and **GitLab single sign-on URL**.
+1. Configure your SAML identity provider app using the noted details.
+   Alternatively, GitLab provides a [metadata XML configuration](#metadata-configuration).
    See [specific identity provider documentation](#providers) for more details.
-1. Configure the SAML response to include a NameID that uniquely identifies each user.
+1. Configure the SAML response to include a [NameID](#nameid) that uniquely identifies each user.
 1. Configure the required [user attributes](#user-attributes), ensuring you include the user's email address.
 1. While the default is enabled for most SAML providers, please ensure the app is set to have service provider
    initiated calls in order to link existing GitLab accounts.

doc/user/group/saml_sso/scim_setup.md

@@ -20,7 +20,7 @@ The GitLab [SCIM API](../../../api/scim.md) implements part of [the RFC7644 prot
 The following actions are available:
 
 - Create users
-- Deactivate users
+- Remove users (deactivate SCIM identity)
 
 The following identity providers are supported:
 

doc/user/project/merge_requests/status_checks.md

@@ -54,6 +54,9 @@ External status checks have the following states:
 
 Support for adding a `failed` state is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338827).
 
+If something changes outside of GitLab, you can manually [set the status external status check](../../../api/status_checks.md#set-status-of-an-external-status-check)
+using the API. You don't need to wait for a merge request webhook payload to be sent first.
+
 ## View the status checks on a project
 
 Within each project's settings, you can see a list of status checks added to the project:

lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb

@@ -22,8 +22,6 @@ module Gitlab
       def perform(start_id, end_id)
         eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id)
 
-        return if eligible_mrs.empty?
-
         eligible_mrs.each_slice(10) do |slice|
           MergeRequest.where(id: slice).update_all(draft: true)
         end

lib/gitlab/usage_data_counters/known_events/epic_events.yml

@@ -200,3 +200,9 @@
   redis_slot: project_management
   aggregation: daily
   feature_flag: track_epics_activity
+
+- name: g_project_management_epic_blocking_added
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity

locale/gitlab.pot

@@ -1557,9 +1557,6 @@ msgstr ""
 msgid "A Let's Encrypt SSL certificate can not be obtained until your domain is verified."
 msgstr ""
 
-msgid "A Metrics Dashboard menu item appears in the Monitoring section of the Admin Area."
-msgstr ""
-
 msgid "A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages"
 msgstr ""
 
@@ -2127,9 +2124,6 @@ msgstr ""
 msgid "Add a link"
 msgstr ""
 
-msgid "Add a link to Grafana"
-msgstr ""
-
 msgid "Add a new issue"
 msgstr ""
 
@@ -4340,6 +4334,12 @@ msgstr ""
 msgid "Application: %{name}"
 msgstr ""
 
+msgid "ApplicationSettings|A Metrics Dashboard menu item appears in the Monitoring section of the Admin Area."
+msgstr ""
+
+msgid "ApplicationSettings|Add a link to Grafana"
+msgstr ""
+
 msgid "ApplicationSettings|After sign up text"
 msgstr ""
 

spec/controllers/jira_connect/subscriptions_controller_spec.rb

@@ -75,6 +75,18 @@ RSpec.describe JiraConnect::SubscriptionsController do
             expect(json_response).to include('login_path' => nil)
           end
         end
+
+        context 'with context qsh' do
+          # The JSON endpoint will be requested by frontend using a JWT that Atlassian provides via Javascript.
+          # This JWT will likely use a context-qsh because Atlassian don't know for which endpoint it will be used.
+          # Read more about context JWT here: https://developer.atlassian.com/cloud/jira/platform/understanding-jwt-for-connect-apps/
+
+          let(:qsh) { 'context-qsh' }
+
+          specify do
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+        end
       end
     end
   end
@@ -102,7 +114,7 @@ RSpec.describe JiraConnect::SubscriptionsController do
     end
 
     context 'with valid JWT' do
-      let(:claims) { { iss: installation.client_key, sub: 1234 } }
+      let(:claims) { { iss: installation.client_key, sub: 1234, qsh: '123' } }
       let(:jwt) { Atlassian::Jwt.encode(claims, installation.shared_secret) }
       let(:jira_user) { { 'groups' => { 'items' => [{ 'name' => jira_group_name }] } } }
       let(:jira_group_name) { 'site-admins' }
@@ -158,7 +170,7 @@ RSpec.describe JiraConnect::SubscriptionsController do
         .stub_request(:get, "#{installation.base_url}/rest/api/3/user?accountId=1234&expand=groups")
         .to_return(body: jira_user.to_json, status: 200, headers: { 'Content-Type' => 'application/json' })
 
-      delete :destroy, params: { jwt: jwt, id: subscription.id }
+      delete :destroy, params: { jwt: jwt, id: subscription.id, format: :json }
     end
 
     context 'without JWT' do
@@ -170,7 +182,7 @@ RSpec.describe JiraConnect::SubscriptionsController do
     end
 
     context 'with valid JWT' do
-      let(:claims) { { iss: installation.client_key, sub: 1234 } }
+      let(:claims) { { iss: installation.client_key, sub: 1234, qsh: '123' } }
       let(:jwt) { Atlassian::Jwt.encode(claims, installation.shared_secret) }
 
       it 'deletes the subscription' do

spec/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_spec.rb

@@ -50,5 +50,19 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillDraftStatusOnMergeRequests,
 
       subject.perform(mr_ids.first, mr_ids.last)
     end
+
+    it_behaves_like 'marks background migration job records' do
+      let!(:non_eligible_mrs) do
+        Array.new(2) do
+          create_merge_request(
+            title: "Not a d-r-a-f-t 1",
+            draft: false,
+            state_id: 1
+          )
+        end
+      end
+
+      let(:arguments) { [non_eligible_mrs.first.id, non_eligible_mrs.last.id] }
+    end
   end
 end

workhorse/internal/api/api.go

@@ -64,7 +64,13 @@ func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API
 }
 
 type GeoProxyEndpointResponse struct {
-	GeoProxyURL string `json:"geo_proxy_url"`
+	GeoProxyURL       string `json:"geo_proxy_url"`
+	GeoProxyExtraData string `json:"geo_proxy_extra_data"`
+}
+
+type GeoProxyData struct {
+	GeoProxyURL       *url.URL
+	GeoProxyExtraData string
 }
 
 type HandleFunc func(http.ResponseWriter, *http.Request, *Response)
@@ -394,7 +400,7 @@ func validResponseContentType(resp *http.Response) bool {
 	return helper.IsContentType(ResponseContentType, resp.Header.Get("Content-Type"))
 }
 
-func (api *API) GetGeoProxyURL() (*url.URL, error) {
+func (api *API) GetGeoProxyData() (*GeoProxyData, error) {
 	geoProxyApiUrl := *api.URL
 	geoProxyApiUrl.Path, geoProxyApiUrl.RawPath = joinURLPath(api.URL, geoProxyEndpointPath)
 	geoProxyApiReq := &http.Request{
@@ -405,23 +411,26 @@ func (api *API) GetGeoProxyURL() (*url.URL, error) {
 
 	httpResponse, err := api.doRequestWithoutRedirects(geoProxyApiReq)
 	if err != nil {
-		return nil, fmt.Errorf("GetGeoProxyURL: do request: %v", err)
+		return nil, fmt.Errorf("GetGeoProxyData: do request: %v", err)
 	}
 	defer httpResponse.Body.Close()
 
 	if httpResponse.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("GetGeoProxyURL: Received HTTP status code: %v", httpResponse.StatusCode)
+		return nil, fmt.Errorf("GetGeoProxyData: Received HTTP status code: %v", httpResponse.StatusCode)
 	}
 
 	response := &GeoProxyEndpointResponse{}
 	if err := json.NewDecoder(httpResponse.Body).Decode(response); err != nil {
-		return nil, fmt.Errorf("GetGeoProxyURL: decode response: %v", err)
+		return nil, fmt.Errorf("GetGeoProxyData: decode response: %v", err)
 	}
 
 	geoProxyURL, err := url.Parse(response.GeoProxyURL)
 	if err != nil {
-		return nil, fmt.Errorf("GetGeoProxyURL: Could not parse Geo proxy URL: %v, err: %v", response.GeoProxyURL, err)
+		return nil, fmt.Errorf("GetGeoProxyData: Could not parse Geo proxy URL: %v, err: %v", response.GeoProxyURL, err)
 	}
 
-	return geoProxyURL, nil
+	return &GeoProxyData{
+		GeoProxyURL:       geoProxyURL,
+		GeoProxyExtraData: response.GeoProxyExtraData,
+	}, nil
 }

workhorse/internal/api/api_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"regexp"
 	"testing"
 
@@ -18,21 +17,37 @@ import (
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper"
 )
 
-func TestGetGeoProxyURLWhenGeoSecondary(t *testing.T) {
-	geoProxyURL, err := getGeoProxyURLGivenResponse(t, `{"geo_proxy_url":"http://primary"}`)
+func TestGetGeoProxyDataForResponses(t *testing.T) {
+	testCases := []struct {
+		desc              string
+		json              string
+		expectedError     bool
+		expectedURL       string
+		expectedExtraData string
+	}{
+		{"when Geo secondary", `{"geo_proxy_url":"http://primary","geo_proxy_extra_data":"geo-data"}`, false, "http://primary", "geo-data"},
+		{"when Geo secondary with explicit null data", `{"geo_proxy_url":"http://primary","geo_proxy_extra_data":null}`, false, "http://primary", ""},
+		{"when Geo secondary without extra data", `{"geo_proxy_url":"http://primary"}`, false, "http://primary", ""},
+		{"when Geo primary or no node", `{}`, false, "", ""},
+		{"for malformed request", `non-json`, true, "", ""},
+	}
 
-	require.NoError(t, err)
-	require.Equal(t, "http://primary", geoProxyURL.String())
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			geoProxyData, err := getGeoProxyDataGivenResponse(t, tc.json)
+
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedURL, geoProxyData.GeoProxyURL.String())
+				require.Equal(t, tc.expectedExtraData, geoProxyData.GeoProxyExtraData)
+			}
+		})
+	}
 }
 
-func TestGetGeoProxyURLWhenGeoPrimaryOrNonGeo(t *testing.T) {
-	geoProxyURL, err := getGeoProxyURLGivenResponse(t, "{}")
-
-	require.NoError(t, err)
-	require.Equal(t, "", geoProxyURL.String())
-}
-
-func getGeoProxyURLGivenResponse(t *testing.T, givenInternalApiResponse string) (*url.URL, error) {
+func getGeoProxyDataGivenResponse(t *testing.T, givenInternalApiResponse string) (*GeoProxyData, error) {
 	t.Helper()
 	ts := testRailsServer(regexp.MustCompile(`/api/v4/geo/proxy`), 200, givenInternalApiResponse)
 	defer ts.Close()
@@ -43,9 +58,9 @@ func getGeoProxyURLGivenResponse(t *testing.T, givenInternalApiResponse string)
 
 	apiClient := NewAPI(backend, version, rt)
 
-	geoProxyURL, err := apiClient.GetGeoProxyURL()
+	geoProxyData, err := apiClient.GetGeoProxyData()
 
-	return geoProxyURL, err
+	return geoProxyData, err
 }
 
 func testRailsServer(url *regexp.Regexp, code int, body string) *httptest.Server {

workhorse/internal/upstream/upstream.go

@@ -37,7 +37,6 @@ var (
 		upload.RewrittenFieldsHeader,
 	}
 	geoProxyApiPollingInterval = 10 * time.Second
-	geoProxyWorkhorseHeaders   = map[string]string{"Gitlab-Workhorse-Geo-Proxy": "1"}
 )
 
 type upstream struct {
@@ -48,6 +47,7 @@ type upstream struct {
 	CableRoundTripper     http.RoundTripper
 	APIClient             *apipkg.API
 	geoProxyBackend       *url.URL
+	geoProxyExtraData     string
 	geoLocalRoutes        []routeEntry
 	geoProxyCableRoute    routeEntry
 	geoProxyRoute         routeEntry
@@ -215,28 +215,44 @@ func (u *upstream) pollGeoProxyAPI() {
 
 // Calls /api/v4/geo/proxy and sets up routes
 func (u *upstream) callGeoProxyAPI() {
-	geoProxyURL, err := u.APIClient.GetGeoProxyURL()
+	geoProxyData, err := u.APIClient.GetGeoProxyData()
 	if err != nil {
 		log.WithError(err).WithFields(log.Fields{"geoProxyBackend": u.geoProxyBackend}).Error("Geo Proxy: Unable to determine Geo Proxy URL. Fallback on cached value.")
 		return
 	}
 
-	if u.geoProxyBackend.String() != geoProxyURL.String() {
-		log.WithFields(log.Fields{"oldGeoProxyURL": u.geoProxyBackend, "newGeoProxyURL": geoProxyURL}).Info("Geo Proxy: URL changed")
-		u.updateGeoProxyFields(geoProxyURL)
+	hasProxyDataChanged := false
+	if u.geoProxyBackend.String() != geoProxyData.GeoProxyURL.String() {
+		log.WithFields(log.Fields{"oldGeoProxyURL": u.geoProxyBackend, "newGeoProxyURL": geoProxyData.GeoProxyURL}).Info("Geo Proxy: URL changed")
+		hasProxyDataChanged = true
+	}
+
+	if u.geoProxyExtraData != geoProxyData.GeoProxyExtraData {
+		// extra data is usually a JWT, thus not explicitly logging it
+		log.Info("Geo Proxy: signed data changed")
+		hasProxyDataChanged = true
+	}
+
+	if hasProxyDataChanged {
+		u.updateGeoProxyFieldsFromData(geoProxyData)
 	}
 }
 
-func (u *upstream) updateGeoProxyFields(geoProxyURL *url.URL) {
+func (u *upstream) updateGeoProxyFieldsFromData(geoProxyData *apipkg.GeoProxyData) {
 	u.mu.Lock()
 	defer u.mu.Unlock()
 
-	u.geoProxyBackend = geoProxyURL
+	u.geoProxyBackend = geoProxyData.GeoProxyURL
+	u.geoProxyExtraData = geoProxyData.GeoProxyExtraData
 
 	if u.geoProxyBackend.String() == "" {
 		return
 	}
 
+	geoProxyWorkhorseHeaders := map[string]string{
+		"Gitlab-Workhorse-Geo-Proxy":            "1",
+		"Gitlab-Workhorse-Geo-Proxy-Extra-Data": u.geoProxyExtraData,
+	}
 	geoProxyRoundTripper := roundtripper.NewBackendRoundTripper(u.geoProxyBackend, "", u.ProxyHeadersTimeout, u.DevelopmentMode)
 	geoProxyUpstream := proxypkg.NewProxy(
 		u.geoProxyBackend,

workhorse/internal/upstream/upstream_test.go

@@ -209,21 +209,74 @@ func TestGeoProxyFeatureEnablingAndDisabling(t *testing.T) {
 	runTestCases(t, ws, testCasesProxied)
 }
 
-func TestGeoProxySetsCustomHeader(t *testing.T) {
+func TestGeoProxyUpdatesExtraDataWhenChanged(t *testing.T) {
+	var expectedGeoProxyExtraData string
+
 	remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "1", r.Header.Get("Gitlab-Workhorse-Geo-Proxy"), "custom proxy header")
+		require.Equal(t, expectedGeoProxyExtraData, r.Header.Get("Gitlab-Workhorse-Geo-Proxy-Extra-Data"), "custom extra data header")
 		w.WriteHeader(http.StatusOK)
 	}))
 	defer remoteServer.Close()
 
-	geoProxyEndpointResponseBody := fmt.Sprintf(`{"geo_proxy_url":"%v"}`, remoteServer.URL)
+	geoProxyEndpointExtraData1 := fmt.Sprintf(`{"geo_proxy_url":"%v","geo_proxy_extra_data":"data1"}`, remoteServer.URL)
+	geoProxyEndpointExtraData2 := fmt.Sprintf(`{"geo_proxy_url":"%v","geo_proxy_extra_data":"data2"}`, remoteServer.URL)
+	geoProxyEndpointExtraData3 := fmt.Sprintf(`{"geo_proxy_url":"%v"}`, remoteServer.URL)
+	geoProxyEndpointResponseBody := geoProxyEndpointExtraData1
+	expectedGeoProxyExtraData = "data1"
+
 	railsServer, deferredClose := startRailsServer("Local Rails server", &geoProxyEndpointResponseBody)
 	defer deferredClose()
 
-	ws, wsDeferredClose, _ := startWorkhorseServer(railsServer.URL, true)
+	ws, wsDeferredClose, waitForNextApiPoll := startWorkhorseServer(railsServer.URL, true)
 	defer wsDeferredClose()
 
 	http.Get(ws.URL)
+
+	// Verify that the expected header changes after next updated poll.
+	geoProxyEndpointResponseBody = geoProxyEndpointExtraData2
+	expectedGeoProxyExtraData = "data2"
+	waitForNextApiPoll()
+
+	http.Get(ws.URL)
+
+	// Validate that non-existing extra data results in empty header
+	geoProxyEndpointResponseBody = geoProxyEndpointExtraData3
+	expectedGeoProxyExtraData = ""
+	waitForNextApiPoll()
+
+	http.Get(ws.URL)
+}
+
+func TestGeoProxySetsCustomHeader(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		json      string
+		extraData string
+	}{
+		{"no extra data", `{"geo_proxy_url":"%v"}`, ""},
+		{"with extra data", `{"geo_proxy_url":"%v","geo_proxy_extra_data":"extra-geo-data"}`, "extra-geo-data"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, "1", r.Header.Get("Gitlab-Workhorse-Geo-Proxy"), "custom proxy header")
+				require.Equal(t, tc.extraData, r.Header.Get("Gitlab-Workhorse-Geo-Proxy-Extra-Data"), "custom proxy extra data header")
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer remoteServer.Close()
+
+			geoProxyEndpointResponseBody := fmt.Sprintf(tc.json, remoteServer.URL)
+			railsServer, deferredClose := startRailsServer("Local Rails server", &geoProxyEndpointResponseBody)
+			defer deferredClose()
+
+			ws, wsDeferredClose, _ := startWorkhorseServer(railsServer.URL, true)
+			defer wsDeferredClose()
+
+			http.Get(ws.URL)
+		})
+	}
 }
 
 func runTestCases(t *testing.T, ws *httptest.Server, testCases []testCase) {