Add latest changes from gitlab-org/gitlab@master

app/models/concerns/ignorable_columns.rb

@@ -31,15 +31,13 @@ module IgnorableColumns
     alias_method :ignore_column, :ignore_columns
 
     def ignored_columns_details
-      unless defined?(@ignored_columns_details)
-        IGNORE_COLUMN_MUTEX.synchronize do
-          @ignored_columns_details ||= superclass.try(:ignored_columns_details)&.dup || {}
-        end
-      end
+      return @ignored_columns_details if defined?(@ignored_columns_details)
 
-      @ignored_columns_details
+      IGNORE_COLUMN_MONITOR.synchronize do
+        @ignored_columns_details ||= superclass.try(:ignored_columns_details)&.dup || {}
+      end
     end
 
-    IGNORE_COLUMN_MUTEX = Mutex.new
+    IGNORE_COLUMN_MONITOR = Monitor.new
   end
 end

app/models/concerns/issuable.rb

@@ -197,7 +197,7 @@ module Issuable
     end
 
     def severity
-      return IssuableSeverity::DEFAULT unless incident?
+      return IssuableSeverity::DEFAULT unless supports_severity?
 
       issuable_severity&.severity || IssuableSeverity::DEFAULT
     end

app/services/incident_management/incidents/update_severity_service.rb

@@ -12,7 +12,7 @@ module IncidentManagement
       end
 
       def execute
-        return unless issuable.incident?
+        return unless issuable.supports_severity?
 
         update_severity!
         add_system_note

changelogs/unreleased/205578-adds-guest-pkg-events-to-usage-data.yml

@@ -0,0 +1,5 @@
+---
+title: Adds guest package events to usage data
+merge_request: 48734
+author:
+type: added

doc/.vale/gitlab/spelling-exceptions.txt

@@ -349,6 +349,8 @@ Prettifier
 Pritaly
 profiler
 Prometheus
+protobuf
+protobufs
 proxied
 proxies
 proxyable

doc/development/agent/routing.md

@@ -0,0 +1,223 @@
+---
+stage: Configure
+group: Configure
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
+---
+
+# Routing `kas` requests in the Kubernetes Agent **(PREMIUM ONLY)**
+
+This document describes how `kas` routes requests to concrete `agentk` instances.
+GitLab must talk to GitLab Kubernetes Agent Server (`kas`) to:
+
+- Get information about connected agents. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/249560).
+- Interact with agents. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/230571).
+- Interact with Kubernetes clusters. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/240918).
+
+Each agent connects to an instance of `kas` and keeps an open connection. When
+GitLab must talk to a particular agent, a `kas` instance connected to this agent must
+be found, and the request routed to it.
+
+## System design
+
+For an architecture overview please see
+[architecture.md](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md).
+
+```mermaid
+flowchart LR
+  subgraph "Kubernetes 1"
+    agentk1p1["agentk 1, Pod1"]
+    agentk1p2["agentk 1, Pod2"]
+  end
+
+  subgraph "Kubernetes 2"
+    agentk2p1["agentk 2, Pod1"]
+  end
+
+  subgraph "Kubernetes 3"
+    agentk3p1["agentk 3, Pod1"]
+  end
+
+  subgraph kas
+    kas1["kas 1"]
+    kas2["kas 2"]
+    kas3["kas 3"]
+  end
+
+  GitLab["GitLab Rails"]
+  Redis
+
+  GitLab -- "gRPC to any kas" --> kas
+  kas1 -- register connected agents --> Redis
+  kas2 -- register connected agents --> Redis
+  kas1 -- lookup agent --> Redis
+
+  agentk1p1 -- "gRPC" --> kas1
+  agentk1p2 -- "gRPC" --> kas2
+  agentk2p1 -- "gRPC" --> kas1
+  agentk3p1 -- "gRPC" --> kas2
+```
+
+For this architecture, this diagram shows a request to `agentk 3, Pod1` for the list of pods:
+
+```mermaid
+sequenceDiagram
+  GitLab->>+kas1: Get list of running<br />Pods from agentk<br />with agent_id=3
+  Note right of kas1: kas1 checks for<br />agent connected with agent_id=3.<br />It does not.<br />Queries Redis
+  kas1->>+Redis: Get list of connected agents<br />with agent_id=3
+  Redis-->-kas1: List of connected agents<br />with agent_id=3
+  Note right of kas1: kas1 picks a specific agentk instance<br />to address and talks to<br />the corresponding kas instance,<br />specifying which agentk instance<br />to route the request to.
+  kas1->>+kas2: Get the list of running Pods<br />from agentk 3, Pod1
+  kas2->>+agentk 3 Pod1: Get list of Pods
+  agentk 3 Pod1->>-kas2: Get list of Pods
+  kas2-->>-kas1: List of running Pods<br />from agentk 3, Pod1
+  kas1-->>-GitLab: List of running Pods<br />from agentk with agent_id=3
+```
+
+Each `kas` instance tracks the agents connected to it in Redis. For each agent, it
+stores a serialized protobuf object with information about the agent. When an agent
+disconnects, `kas` removes all corresponding information from Redis. For both events,
+`kas` publishes a notification to a Redis [pub-sub channel](https://redis.io/topics/pubsub).
+
+Each agent, while logically a single entity, can have multiple replicas (multiple pods)
+in a cluster. `kas` accommodates that and records per-replica (generally per-connection)
+information. Each open `GetConfiguration()` streaming request is given
+a unique identifier which, combined with agent ID, identifies an `agentk` instance.
+
+gRPC can keep multiple TCP connections open for a single target host. `agentk` only
+runs one `GetConfiguration()` streaming request. `kas` uses that connection, and
+doesn't see idle TCP connections because they are handled by the gRPC framework.
+
+Each `kas` instance provides information to Redis, so other `kas` instances can discover and access it.
+
+Information is stored in Redis with an [expiration time](https://redis.io/commands/expire),
+to expire information for `kas` instances that become unavailable. To prevent
+information from expiring too quickly, `kas` periodically updates the expiration time
+for valid entries. Before terminating, `kas` cleans up the information it adds into Redis.
+
+When `kas` must atomically update multiple data structures in Redis, it uses
+[transactions](https://redis.io/topics/transactions) to ensure data consistency.
+Grouped data items must have the same expiration time.
+
+In addition to the existing `agentk -> kas` gRPC endpoint, `kas` exposes two new,
+separate gRPC endpoints for GitLab and for `kas -> kas` requests. Each endpoint
+is a separate network listener, making it easier to control network access to endpoints
+and allowing separate configuration for each endpoint.
+
+Databases, like PostgreSQL, aren't used because the data is transient, with no need
+to reliably persist it.
+
+### `GitLab : kas` external endpoint
+
+GitLab authenticates with `kas` using JWT and the same shared secret used by the
+`kas -> GitLab` communication. The JWT issuer should be `gitlab` and the audience
+should be `gitlab-kas`.
+
+When accessed through this endpoint, `kas` plays the role of request router.
+
+If a request from GitLab comes but no connected agent can handle it, `kas` blocks
+and waits for a suitable agent to connect to it or to another `kas` instance. It
+stops waiting when the client disconnects, or when some long timeout happens, such
+as client timeout. `kas` is notified of new agent connections through a
+[pub-sub channel](https://redis.io/topics/pubsub) to avoid frequent polling.
+When a suitable agent connects, `kas` routes the request to it.
+
+### `kas : kas` internal endpoint
+
+This endpoint is an implementation detail, an internal API, and should not be used
+by any other system. It's protected by JWT using a secret, shared among all `kas`
+instances. No other system must have access to this secret.
+
+When accessed through this endpoint, `kas` uses the request itself to determine
+which `agentk` to send the request to. It prevents request cycles by only following
+the instructions in the request, rather than doing discovery. It's the responsibility
+of the `kas` receiving the request from the _external_ endpoint to retry and re-route
+requests. This method ensures a single central component for each request can determine
+how a request is routed, rather than distributing the decision across several `kas` instances.
+
+### API definitions
+
+```proto
+syntax = "proto3";
+
+import "google/protobuf/timestamp.proto";
+
+message KasAddress {
+    string ip = 1;
+    uint32 port = 2;
+}
+
+message ConnectedAgentInfo {
+    // Agent id.
+    int64 id = 1;
+    // Identifies a particular agentk->kas connection. Randomly generated when agent connects.
+    int64 connection_id = 2;
+    string version = 3;
+    string commit = 4;
+    // Pod namespace.
+    string pod_namespace = 5;
+    // Pod name.
+    string pod_name = 6;
+    // When the connection was established.
+    google.protobuf.Timestamp connected_at = 7;
+    KasAddress kas_address = 8;
+    // What else do we need?
+}
+
+message KasInstanceInfo {
+    string version = 1;
+    string commit = 2;
+    KasAddress address = 3;
+    // What else do we need?
+}
+
+message ConnectedAgentsForProjectRequest {
+    int64 project_id = 1;
+}
+
+message ConnectedAgentsForProjectResponse {
+    // There may 0 or more agents with the same id, depending on the number of running Pods.
+    repeated ConnectedAgentInfo agents = 1;
+}
+
+message ConnectedAgentsByIdRequest {
+    int64 agent_id = 1;
+}
+
+message ConnectedAgentsByIdResponse {
+    repeated ConnectedAgentInfo agents = 1;
+}
+
+// API for use by GitLab.
+service KasApi {
+    // Connected agents for a particular configuration project.
+    rpc ConnectedAgentsForProject (ConnectedAgentsForProjectRequest) returns (ConnectedAgentsForProjectResponse) {
+    }
+    // Connected agents for a particular agent id.
+    rpc ConnectedAgentsById (ConnectedAgentsByIdRequest) returns (ConnectedAgentsByIdResponse) {
+    }
+    // Depends on the need, but here is the call from the example above.
+    rpc GetPods (GetPodsRequest) returns (GetPodsResponse) {
+    }
+}
+
+message Pod {
+    string namespace = 1;
+    string name = 2;
+}
+
+message GetPodsRequest {
+    int64 agent_id = 1;
+    int64 connection_id = 2;
+}
+
+message GetPodsResponse {
+    repeated Pod pods = 1;
+}
+
+// Internal API for use by kas for kas -> kas calls.
+service KasInternal {
+    // Depends on the need, but here is the call from the example above.
+    rpc GetPods (GetPodsRequest) returns (GetPodsResponse) {
+    }
+}
+```

doc/development/testing_guide/end_to_end/environment_selection.md

@@ -35,7 +35,7 @@ can control the `tld` and `domain` independently.
 | `dev.gitlab.org`                         | `only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' }` | `(dev).gitlab.org` |
 | `staging.gitlab.com & domain.gitlab.com` | `only: { subdomain: %i[staging domain] }` | `(staging|domain).+.com`             |
 | `nightly`                                | `only: { pipeline: :nightly }` | "nightly" |
-| `nightly`, `canary` | `only_run_in_pipeline: [:nightly, :canary]` | ["nightly"](https://gitlab.com/gitlab-org/quality/nightly) and ["canary"](https://gitlab.com/gitlab-org/quality/canary) |
+| `nightly`, `canary` | `only: { pipeline: [:nightly, :canary] }` | ["nightly"](https://gitlab.com/gitlab-org/quality/nightly) and ["canary"](https://gitlab.com/gitlab-org/quality/canary) |
 
 ```ruby
 RSpec.describe 'Area' do

doc/user/project/import/gemnasium.md

@@ -66,19 +66,19 @@ GitHub.com or GitHub Enterprise repository. This will automatically prompt
 GitLab CI/CD to run whenever code is pushed to GitHub and post CI/CD results
 back to both GitLab and GitHub when completed.
 
-1. Create a new project, and select the "CI/CD for external repo" tab:
+1. Create a new project, and select "CI/CD for external repo":
 
-   ![Create new Project](img/gemnasium/create_project.png)
+   ![Create new Project](img/gemnasium/create_project_v13_5.png)
 
 1. Use the "GitHub" button to connect your repositories.
 
-   ![Connect from GitHub](img/gemnasium/connect_github.png)
+   ![Connect from GitHub](img/gemnasium/connect_github_v13_5.png)
 
 1. Select the project(s) to be set up with GitLab CI/CD and chose "Connect".
 
-   ![Select projects](img/gemnasium/select_project.png)
+   ![Select projects](img/gemnasium/select_project_v13_5.png)
 
-   Once the configuration is done, you may click on your new
+   After the configuration is done, you may click on your new
    project on GitLab.
 
    ![click on connected project](img/gemnasium/project_connected.png)

doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md

@@ -18,7 +18,7 @@ View all the merge requests within a project by navigating to **Project > Merge
 When you access your project's merge requests, GitLab will present them in a list,
 and you can use the tabs available to quickly filter by open and closed. You can also [search and filter the results](../../search/index.md#filtering-issue-and-merge-request-lists).
 
-![Project merge requests list view](img/project_merge_requests_list_view.png)
+![Project merge requests list view](img/project_merge_requests_list_view_v13_5.png)
 
 ## View merge requests for all projects in a group
 

doc/user/project/milestones/index.md

@@ -76,7 +76,7 @@ To create a **group milestone**:
 1. Enter the title, an optional description, an optional start date, and an optional due date.
 1. Click **New milestone**.
 
-![New group milestone](img/milestones_new_group_milestone.png)
+![New group milestone](img/milestones_new_group_milestone_v13_5.png)
 
 ## Editing milestones
 

lib/gitlab/usage_data_counters.rb

@@ -3,6 +3,7 @@
 module Gitlab
   module UsageDataCounters
     COUNTERS = [
+      GuestPackageEventCounter,
       WikiPageCounter,
       WebIdeCounter,
       NoteCounter,

qa/qa/specs/features/sanity/version_spec.rb

@@ -7,7 +7,7 @@ module QA
   # environment variable is the version actually running.
   #
   # See https://gitlab.com/gitlab-com/gl-infra/delivery/-/issues/1179
-  RSpec.describe 'Version sanity check', :smoke do
+  RSpec.describe 'Version sanity check', :smoke, only: { pipeline: [:pre, :release] } do
     let(:api_client) { Runtime::API::Client.new(:gitlab) }
     let(:request) { Runtime::API::Request.new(api_client, '/version') }
 

spec/controllers/repositories/git_http_controller_spec.rb

@@ -9,120 +9,6 @@ RSpec.describe Repositories::GitHttpController do
   let_it_be(:personal_snippet) { create(:personal_snippet, :public, :repository) }
   let_it_be(:project_snippet) { create(:project_snippet, :public, :repository, project: project) }
 
-  shared_examples Repositories::GitHttpController do
-    let(:repository_path) { "#{container.full_path}.git" }
-    let(:params) { { repository_path: repository_path } }
-
-    describe 'HEAD #info_refs' do
-      it 'returns 403' do
-        head :info_refs, params: params
-
-        expect(response).to have_gitlab_http_status(:forbidden)
-      end
-    end
-
-    describe 'GET #info_refs' do
-      let(:params) { super().merge(service: 'git-upload-pack') }
-
-      it 'returns 401 for unauthenticated requests to public repositories when http protocol is disabled' do
-        stub_application_setting(enabled_git_access_protocol: 'ssh')
-        allow(controller).to receive(:basic_auth_provided?).and_call_original
-
-        expect(controller).to receive(:http_download_allowed?).and_call_original
-
-        get :info_refs, params: params
-
-        expect(response).to have_gitlab_http_status(:unauthorized)
-      end
-
-      it 'calls the right access checker class with the right object' do
-        allow(controller).to receive(:verify_workhorse_api!).and_return(true)
-
-        access_double = double
-        options = {
-          authentication_abilities: [:download_code],
-          repository_path: repository_path,
-          redirected_path: nil,
-          auth_result_type: :none
-        }
-
-        expect(access_checker_class).to receive(:new)
-          .with(nil, container, 'http', hash_including(options))
-          .and_return(access_double)
-
-        allow(access_double).to receive(:check).and_return(false)
-
-        get :info_refs, params: params
-      end
-
-      context 'with authorized user' do
-        before do
-          request.headers.merge! auth_env(user.username, user.password, nil)
-        end
-
-        it 'returns 200' do
-          get :info_refs, params: params
-
-          expect(response).to have_gitlab_http_status(:ok)
-        end
-
-        it 'updates the user activity' do
-          expect_next_instance_of(Users::ActivityService) do |activity_service|
-            expect(activity_service).to receive(:execute)
-          end
-
-          get :info_refs, params: params
-        end
-
-        include_context 'parsed logs' do
-          it 'adds user info to the logs' do
-            get :info_refs, params: params
-
-            expect(log_data).to include('username' => user.username,
-                                        'user_id' => user.id,
-                                        'meta.user' => user.username)
-          end
-        end
-      end
-
-      context 'with exceptions' do
-        before do
-          allow(controller).to receive(:authenticate_user).and_return(true)
-          allow(controller).to receive(:verify_workhorse_api!).and_return(true)
-        end
-
-        it 'returns 503 with GRPC Unavailable' do
-          allow(controller).to receive(:access_check).and_raise(GRPC::Unavailable)
-
-          get :info_refs, params: params
-
-          expect(response).to have_gitlab_http_status(:service_unavailable)
-        end
-
-        it 'returns 503 with timeout error' do
-          allow(controller).to receive(:access_check).and_raise(Gitlab::GitAccess::TimeoutError)
-
-          get :info_refs, params: params
-
-          expect(response).to have_gitlab_http_status(:service_unavailable)
-          expect(response.body).to eq 'Gitlab::GitAccess::TimeoutError'
-        end
-      end
-    end
-
-    describe 'POST #git_upload_pack' do
-      before do
-        allow(controller).to receive(:verify_workhorse_api!).and_return(true)
-      end
-
-      it 'returns 200' do
-        post :git_upload_pack, params: params
-
-        expect(response).to have_gitlab_http_status(:ok)
-      end
-    end
-  end
-
   context 'when repository container is a project' do
     it_behaves_like Repositories::GitHttpController do
       let(:container) { project }

spec/lib/gitlab/usage_data_spec.rb

@@ -654,6 +654,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
 
     it { is_expected.to include(:kubernetes_agent_gitops_sync) }
     it { is_expected.to include(:static_site_editor_views) }
+    it { is_expected.to include(:package_guest_i_package_composer_guest_pull) }
   end
 
   describe '.usage_data_counters' do

spec/models/concerns/ignorable_columns_spec.rb

@@ -59,6 +59,14 @@ RSpec.describe IgnorableColumns do
       it_behaves_like 'storing removal information'
     end
 
+    context 'when called on a subclass without setting the ignored columns' do
+      let(:subclass) { Class.new(record_class) }
+
+      it 'does not raise Deadlock error' do
+        expect { subclass.ignored_columns_details }.not_to raise_error
+      end
+    end
+
     it 'defaults to empty Hash' do
       expect(subject.ignored_columns_details).to eq({})
     end

spec/support/shared_examples/controllers/repositories/git_http_controller_shared_examples.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples Repositories::GitHttpController do
+  include GitHttpHelpers
+
+  let(:repository_path) { "#{container.full_path}.git" }
+  let(:params) { { repository_path: repository_path } }
+
+  describe 'HEAD #info_refs' do
+    it 'returns 403' do
+      head :info_refs, params: params
+
+      expect(response).to have_gitlab_http_status(:forbidden)
+    end
+  end
+
+  describe 'GET #info_refs' do
+    let(:params) { super().merge(service: 'git-upload-pack') }
+
+    it 'returns 401 for unauthenticated requests to public repositories when http protocol is disabled' do
+      stub_application_setting(enabled_git_access_protocol: 'ssh')
+      allow(controller).to receive(:basic_auth_provided?).and_call_original
+
+      expect(controller).to receive(:http_download_allowed?).and_call_original
+
+      get :info_refs, params: params
+
+      expect(response).to have_gitlab_http_status(:unauthorized)
+    end
+
+    it 'calls the right access checker class with the right object' do
+      allow(controller).to receive(:verify_workhorse_api!).and_return(true)
+
+      access_double = double
+      options = {
+        authentication_abilities: [:download_code],
+        repository_path: repository_path,
+        redirected_path: nil,
+        auth_result_type: :none
+      }
+
+      expect(access_checker_class).to receive(:new)
+        .with(nil, container, 'http', hash_including(options))
+        .and_return(access_double)
+
+      allow(access_double).to receive(:check).and_return(false)
+
+      get :info_refs, params: params
+    end
+
+    context 'with authorized user' do
+      before do
+        request.headers.merge! auth_env(user.username, user.password, nil)
+      end
+
+      it 'returns 200' do
+        get :info_refs, params: params
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+
+      it 'updates the user activity' do
+        expect_next_instance_of(Users::ActivityService) do |activity_service|
+          expect(activity_service).to receive(:execute)
+        end
+
+        get :info_refs, params: params
+      end
+
+      include_context 'parsed logs' do
+        it 'adds user info to the logs' do
+          get :info_refs, params: params
+
+          expect(log_data).to include('username' => user.username,
+                                      'user_id' => user.id,
+                                      'meta.user' => user.username)
+        end
+      end
+    end
+
+    context 'with exceptions' do
+      before do
+        allow(controller).to receive(:authenticate_user).and_return(true)
+        allow(controller).to receive(:verify_workhorse_api!).and_return(true)
+      end
+
+      it 'returns 503 with GRPC Unavailable' do
+        allow(controller).to receive(:access_check).and_raise(GRPC::Unavailable)
+
+        get :info_refs, params: params
+
+        expect(response).to have_gitlab_http_status(:service_unavailable)
+      end
+
+      it 'returns 503 with timeout error' do
+        allow(controller).to receive(:access_check).and_raise(Gitlab::GitAccess::TimeoutError)
+
+        get :info_refs, params: params
+
+        expect(response).to have_gitlab_http_status(:service_unavailable)
+        expect(response.body).to eq 'Gitlab::GitAccess::TimeoutError'
+      end
+    end
+  end
+
+  describe 'POST #git_upload_pack' do
+    before do
+      allow(controller).to receive(:verify_workhorse_api!).and_return(true)
+    end
+
+    it 'returns 200' do
+      post :git_upload_pack, params: params
+
+      expect(response).to have_gitlab_http_status(:ok)
+    end
+  end
+end

spec/support/shared_examples/requests/lfs_http_shared_examples.rb

@@ -65,12 +65,19 @@ end
 RSpec.shared_examples 'LFS http requests' do
   include LfsHttpHelpers
 
+  let(:lfs_enabled) { true }
   let(:authorize_guest) {}
   let(:authorize_download) {}
   let(:authorize_upload) {}
 
   let(:lfs_object) { create(:lfs_object, :with_file) }
   let(:sample_oid) { lfs_object.oid }
+  let(:sample_size) { lfs_object.size }
+  let(:sample_object) { { 'oid' => sample_oid, 'size' => sample_size } }
+  let(:non_existing_object_oid) { '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897' }
+  let(:non_existing_object_size) { 1575078 }
+  let(:non_existing_object) { { 'oid' => non_existing_object_oid, 'size' => non_existing_object_size } }
+  let(:multiple_objects) { [sample_object, non_existing_object] }
 
   let(:authorization) { authorize_user }
   let(:headers) do
@@ -89,13 +96,11 @@ RSpec.shared_examples 'LFS http requests' do
   end
 
   before do
-    stub_lfs_setting(enabled: true)
+    stub_lfs_setting(enabled: lfs_enabled)
   end
 
   context 'when LFS is disabled globally' do
-    before do
-      stub_lfs_setting(enabled: false)
-    end
+    let(:lfs_enabled) { false }
 
     describe 'download request' do
       before do