Add latest changes from gitlab-org/gitlab@master

GITALY_SERVER_VERSION

@@ -1 +1 @@
-dc89f1ebfddca732f55d74488f94a6ed25444b07
+5c57bbfef4f37bcca64e3db2e147265450b86300

app/assets/javascripts/projects/default_project_templates.js

@@ -105,4 +105,8 @@ export default {
     text: s__('ProjectTemplates|Kotlin Native for Linux'),
     icon: '.template-option .icon-gitlab_logo',
   },
+  jsonnet: {
+    text: s__('ProjectTemplates|Jsonnet for Dynamic Child Pipelines'),
+    icon: '.template-option .icon-gitlab_logo',
+  },
 };

app/models/namespace.rb

@@ -427,6 +427,21 @@ class Namespace < ApplicationRecord
     aggregation_schedule.present?
   end
 
+  def container_repositories_size
+    strong_memoize(:container_repositories_size) do
+      next unless Gitlab.com?
+      next unless ContainerRegistry::GitlabApiClient.supports_gitlab_api?
+      next 0 if all_container_repositories.empty?
+      next unless all_container_repositories.all_migrated?
+
+      ContainerRegistry::GitlabApiClient.deduplicated_size(full_path)
+    end
+  end
+
+  def all_container_repositories
+    ContainerRepository.for_project_id(all_projects)
+  end
+
   def pages_virtual_domain
     Pages::VirtualDomain.new(
       all_projects_with_pages.includes(:route, :project_feature, pages_metadatum: :pages_deployment),

app/services/service_ping/submit_service.rb

@@ -10,8 +10,9 @@ module ServicePing
 
     SubmissionError = Class.new(StandardError)
 
-    def initialize(skip_db_write: false)
+    def initialize(skip_db_write: false, payload: nil)
       @skip_db_write = skip_db_write
+      @payload = payload
     end
 
     def execute
@@ -19,7 +20,7 @@ module ServicePing
 
       start = Time.current
       begin
-        usage_data = ServicePing::BuildPayload.new.execute
+        usage_data = payload || ServicePing::BuildPayload.new.execute
         response = submit_usage_data_payload(usage_data)
       rescue StandardError => e
         return unless Gitlab::CurrentSettings.usage_ping_enabled?
@@ -34,7 +35,7 @@ module ServicePing
         }
         submit_payload({ error: error_payload }, path: ERROR_PATH)
 
-        usage_data = Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
+        usage_data = payload || Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
         response = submit_usage_data_payload(usage_data)
       end
 
@@ -45,7 +46,7 @@ module ServicePing
         raise SubmissionError, "Invalid usage_data_id in response: #{version_usage_data_id}"
       end
 
-      unless @skip_db_write
+      unless skip_db_write
         raw_usage_data = save_raw_usage_data(usage_data)
         raw_usage_data.update_version_metadata!(usage_data_id: version_usage_data_id)
         ServicePing::DevopsReport.new(response).execute
@@ -58,6 +59,8 @@ module ServicePing
 
     private
 
+    attr_reader :payload, :skip_db_write
+
     def metrics_collection_time(payload, parents = [])
       return [] unless payload.is_a?(Hash)
 

app/workers/gitlab_service_ping_worker.rb

@@ -25,7 +25,25 @@ class GitlabServicePingWorker # rubocop:disable Scalability/IdempotentWorker
       # Splay the request over a minute to avoid thundering herd problems.
       sleep(rand(0.0..60.0).round(3))
 
-      ServicePing::SubmitService.new.execute
+      ServicePing::SubmitService.new(payload: usage_data).execute
     end
   end
+
+  def usage_data
+    return unless Feature.enabled?(:prerecord_service_ping_data)
+
+    ServicePing::BuildPayload.new.execute.tap do |payload|
+      record = {
+        recorded_at: payload[:recorded_at],
+        payload: payload,
+        created_at: Time.current,
+        updated_at: Time.current
+      }
+
+      RawUsageData.upsert(record, unique_by: :recorded_at)
+    end
+  rescue StandardError => err
+    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err)
+    nil
+  end
 end

config/feature_flags/development/prerecord_service_ping_data.yml

@@ -0,0 +1,8 @@
+---
+name: prerecord_service_ping_data
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85503
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/360675
+milestone: '15.0'
+type: development
+group: group::product intelligence
+default_enabled: true

doc/administration/geo/replication/tuning.md

@@ -7,7 +7,7 @@ type: howto
 
 # Tuning Geo **(PREMIUM SELF)**
 
-You can limit the number of concurrent operations the nodes can run
+You can limit the number of concurrent operations the sites can run
 in the background.
 
 ## Changing the sync/verification concurrency values
@@ -15,8 +15,8 @@ in the background.
 On the **primary** site:
 
 1. On the top bar, select **Menu > Admin**.
-1. On the left sidebar, select **Geo > Nodes**.
-1. Select **Edit** of the secondary node you want to tune.
+1. On the left sidebar, select **Geo > Sites**.
+1. Select **Edit** of the secondary site you want to tune.
 1. Under **Tuning settings**, there are several variables that can be tuned to
    improve the performance of Geo:
 

doc/ci/yaml/index.md

@@ -1016,6 +1016,7 @@ job:
 
 - Scripts you specify in `before_script` are concatenated with any scripts you specify
   in the main [`script`](#script). The combined scripts execute together in a single shell.
+- Using `before_script` at the top level, but not in the `default` section, [is deprecated](#globally-defined-image-services-cache-before_script-after_script).
 
 **Related topics**:
 

doc/development/documentation/index.md

@@ -143,6 +143,71 @@ Nanoc layout), which is displayed at the top of the page if defined.
 The `type` metadata parameter is deprecated but still exists in documentation
 pages. You can safely remove the `type` metadata parameter and its values.
 
+### Batch updates for TW metadata
+
+NOTE:
+This task is an MVC, and requires significant manual preparation of the output.
+While the task can be time consuming, it is still faster than doing the work
+entirely manually.
+
+It's important to keep the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS)
+file in the `gitlab` project up to date with the current Technical Writing team assignments.
+This information is used in merge requests that contain documentation:
+
+- To populate the eligible approvers section.
+- By GitLab Bot to ping reviewers for community contributions.
+
+GitLab cannot automatically associate the stage and group metadata in our documentation
+pages with the technical writer assigned to that group, so we use a Rake task to
+generate entries for the `CODEOWNERS` file. Declaring code owners for pages reduces
+the number of times GitLab Bot pings the entire Technical Writing team.
+
+The `tw:codeowners` Rake task, located in [`lib/tasks/gitlab/tw/codeowners.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake),
+contains an array of groups and their assigned technical writer. This task:
+
+- Outputs a line for each doc with metadata that matches a group in `lib/tasks/gitlab/tw/codeowners.rake`.
+  Files not matching a group are skipped.
+- Adds the full path to the page, and the assigned technical writer.
+
+To prepare an update to the `CODEOWNERS` file:
+
+1. Update `lib/tasks/gitlab/tw/codeowners.rake` with the latest [TW team assignments](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments).
+   Make this update in a standalone merge request, as it runs a long pipeline and
+   requires backend maintainer review. Make sure this is merged before you update
+   `CODEOWNERS` in another merge request.
+1. Run the task from the root directory of the `gitlab` repository, and save the output in a file:
+
+   ```ruby
+   bundle exec rake tw:codeowners > ~/Desktop/updates.md
+   ```
+
+1. Open the file you just created (`~/Desktop/updates.md` in this example), and prepare the output:
+   - Find and replace `./` with `/`.
+   - Sort the lines in alphabetical (ascending) order. If you use VS Code, you can
+     select everything, press <kbd>F1</kbd>, type `sort`, and select **Sort lines (ascending, case insensitive**.
+1. Create a new branch for your `CODEOWNERS` updates.
+1. Replace the documentation-related lines in the `^[Documentation Pages]` section
+   with the output you prepared.
+
+   WARNING:
+   The documentation section is not the last section of the `CODEOWNERS` file. Don't
+   delete data that isn't ours!
+
+1. Create a commit with the raw changes.
+1. From the command line, run `git diff master`.
+1. In the diff, look for directory-level assignments to manually restore to the
+   `CODEOWNERS` file. If all files in a single directory are assigned to the same
+   technical writer, we simplify these entries. Remove all the lines for the individual
+   files, and leave a single entry for the directory, for example: `/doc/directory/ @tech.writer`.
+1. In the diff, look for changes that don't match your expectations:
+   - New pages, or newly moved pages, show up as added lines.
+   - Deleted pages, and pages that are now redirects, show up as deleted lines.
+   - If you see an unusual number of changes to pages that all seem related,
+     check the metadata for the pages. A group might have been renamed and the Rake task
+     must be updated to match.
+1. Create another commit with your manual changes, and create a second merge request
+   with your changes to the `CODEOWNERS` file. Assign it to a technical writing manager for review.
+
 ## Move, rename, or delete a page
 
 See [redirects](redirects.md).

lib/service_ping/build_payload.rb

@@ -3,8 +3,6 @@
 module ServicePing
   class BuildPayload
     def execute
-      return {} unless ServicePingSettings.product_intelligence_enabled?
-
       filtered_usage_data
     end
 

lib/service_ping/permit_data_categories.rb

@@ -14,8 +14,6 @@ module ServicePing
     ].to_set.freeze
 
     def execute
-      return [] unless ServicePingSettings.product_intelligence_enabled?
-
       CATEGORIES
     end
   end

locale/gitlab.pot

@@ -30076,6 +30076,9 @@ msgstr ""
 msgid "ProjectTemplates|HIPAA Audit Protocol"
 msgstr ""
 
+msgid "ProjectTemplates|Jsonnet for Dynamic Child Pipelines"
+msgstr ""
+
 msgid "ProjectTemplates|Kotlin Native for Linux"
 msgstr ""
 

qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb

@@ -233,10 +233,13 @@ module QA
             expect(registry).to have_registry_repository(project.path_with_namespace)
 
             registry.click_on_image(project.path_with_namespace)
+
             expect(registry).to have_tag('master')
 
             registry.click_delete
-            expect(registry).not_to have_tag('master')
+
+            expect { registry.has_no_tag?('master') }
+              .to eventually_be_truthy.within(max_duration: 60, reload_page: page)
           end
         end
       end

qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb

@@ -83,7 +83,8 @@ module QA
           expect(registry).to have_tag('master')
 
           registry.click_delete
-          expect(registry).not_to have_tag('master')
+          expect { registry.has_no_tag?('master') }
+            .to eventually_be_truthy.within(max_duration: 60, reload_page: page)
         end
       end
     end

spec/lib/service_ping/build_payload_spec.rb

@@ -14,35 +14,6 @@ RSpec.describe ServicePing::BuildPayload do
       end
     end
 
-    context 'when usage_ping_enabled setting is false' do
-      before do
-        # Gitlab::CurrentSettings.usage_ping_enabled? == false
-        stub_config_setting(usage_ping_enabled: false)
-      end
-
-      it 'returns empty service ping payload' do
-        expect(service_ping_payload).to eq({})
-      end
-    end
-
-    context 'when usage_ping_enabled setting is true' do
-      before do
-        # Gitlab::CurrentSettings.usage_ping_enabled? == true
-        stub_config_setting(usage_ping_enabled: true)
-      end
-
-      it_behaves_like 'complete service ping payload'
-
-      context 'with require stats consent enabled' do
-        before do
-          allow(User).to receive(:single_user)
-            .and_return(instance_double(User, :user, requires_usage_stats_consent?: true))
-        end
-
-        it 'returns empty service ping payload' do
-          expect(service_ping_payload).to eq({})
-        end
-      end
-    end
+    it_behaves_like 'complete service ping payload'
   end
 end

spec/lib/service_ping/permit_data_categories_spec.rb

@@ -19,26 +19,10 @@ RSpec.describe ServicePing::PermitDataCategories do
     end
 
     context 'when usage ping setting is set to false' do
-      before do
-        allow(User).to receive(:single_user)
-          .and_return(instance_double(User, :user, requires_usage_stats_consent?: false))
+      it 'returns all categories' do
         stub_config_setting(usage_ping_enabled: false)
-      end
 
-      it 'returns no categories' do
-        expect(permitted_categories).to match_array([])
-      end
-    end
-
-    context 'when User.single_user&.requires_usage_stats_consent? is required' do
-      before do
-        allow(User).to receive(:single_user)
-          .and_return(instance_double(User, :user, requires_usage_stats_consent?: true))
-        stub_config_setting(usage_ping_enabled: true)
-      end
-
-      it 'returns no categories' do
-        expect(permitted_categories).to match_array([])
+        expect(permitted_categories).to match_array(%w[standard subscription operational optional])
       end
     end
   end

spec/models/namespace_spec.rb

@@ -574,6 +574,84 @@ RSpec.describe Namespace do
     end
   end
 
+  describe '#container_repositories_size' do
+    let(:project_namespace) { create(:namespace) }
+
+    subject { project_namespace.container_repositories_size }
+
+    context 'on gitlab.com' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:gitlab_api_supported, :no_container_repositories, :all_migrated, :returned_size, :expected_result) do
+        nil   | nil   | nil   | nil | nil
+        false | nil   | nil   | nil | nil
+        true  | true  | nil   | nil | 0
+        true  | false | false | nil | nil
+        true  | false | true  | 555 | 555
+        true  | false | true  | nil | nil
+      end
+
+      with_them do
+        before do
+          stub_container_registry_config(enabled: true, api_url: 'http://container-registry', key: 'spec/fixtures/x509_certificate_pk.key')
+          allow(Gitlab).to receive(:com?).and_return(true)
+          allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(gitlab_api_supported)
+          allow(project_namespace).to receive_message_chain(:all_container_repositories, :empty?).and_return(no_container_repositories)
+          allow(project_namespace).to receive_message_chain(:all_container_repositories, :all_migrated?).and_return(all_migrated)
+          allow(ContainerRegistry::GitlabApiClient).to receive(:deduplicated_size).with(project_namespace.full_path).and_return(returned_size)
+        end
+
+        it { is_expected.to eq(expected_result) }
+      end
+    end
+
+    context 'not on gitlab.com' do
+      it { is_expected.to eq(nil) }
+    end
+  end
+
+  describe '#all_container_repositories' do
+    context 'with personal namespace' do
+      let_it_be(:user) { create(:user) }
+      let_it_be(:project_namespace) { user.namespace }
+
+      context 'with no project' do
+        it { expect(project_namespace.all_container_repositories).to match_array([]) }
+      end
+
+      context 'with projects' do
+        it "returns container repositories" do
+          project = create(:project, namespace: project_namespace)
+          rep = create(:container_repository, project: project)
+
+          expect(project_namespace.all_container_repositories).to match_array([rep])
+        end
+      end
+    end
+
+    context 'with subgroups' do
+      let_it_be(:project_namespace) { create(:group) }
+      let_it_be(:subgroup1) { create(:group, parent: project_namespace) }
+      let_it_be(:subgroup2) { create(:group, parent: subgroup1) }
+
+      context 'with no project' do
+        it { expect(project_namespace.all_container_repositories).to match_array([]) }
+      end
+
+      context 'with projects' do
+        it "returns container repositories" do
+          subgrp1_project = create(:project, namespace: subgroup1)
+          rep1 = create(:container_repository, project: subgrp1_project)
+
+          subgrp2_project = create(:project, namespace: subgroup2)
+          rep2 = create(:container_repository, project: subgrp2_project)
+
+          expect(project_namespace.all_container_repositories).to match_array([rep1, rep2])
+        end
+      end
+    end
+  end
+
   describe '.search' do
     let_it_be(:first_group) { create(:group, name: 'my first namespace', path: 'old-path') }
     let_it_be(:parent_group) { create(:group, name: 'my parent namespace', path: 'parent-path') }

spec/workers/gitlab_service_ping_worker_spec.rb

@@ -3,8 +3,14 @@
 require 'spec_helper'
 
 RSpec.describe GitlabServicePingWorker, :clean_gitlab_redis_shared_state do
+  let(:payload) { { recorded_at: Time.current.rfc3339 } }
+
   before do
     allow_next_instance_of(ServicePing::SubmitService) { |service| allow(service).to receive(:execute) }
+    allow_next_instance_of(ServicePing::BuildPayload) do |service|
+      allow(service).to receive(:execute).and_return(payload)
+    end
+
     allow(subject).to receive(:sleep)
   end
 
@@ -15,10 +21,54 @@ RSpec.describe GitlabServicePingWorker, :clean_gitlab_redis_shared_state do
     subject.perform
   end
 
-  it 'delegates to ServicePing::SubmitService' do
-    expect_next_instance_of(ServicePing::SubmitService) { |service| expect(service).to receive(:execute) }
+  context 'with prerecord_service_ping_data feature enabled' do
+    it 'delegates to ServicePing::SubmitService' do
+      stub_feature_flags(prerecord_service_ping_data: true)
 
-    subject.perform
+      expect_next_instance_of(ServicePing::SubmitService, payload: payload) do |service|
+        expect(service).to receive(:execute)
+      end
+
+      subject.perform
+    end
+  end
+
+  context 'with prerecord_service_ping_data feature disabled' do
+    it 'does not prerecord ServicePing, and calls SubmitService', :aggregate_failures do
+      stub_feature_flags(prerecord_service_ping_data: false)
+
+      expect(ServicePing::BuildPayload).not_to receive(:new)
+      expect(ServicePing::BuildPayload).not_to receive(:new)
+      expect_next_instance_of(ServicePing::SubmitService, payload: nil) do |service|
+        expect(service).to receive(:execute)
+      end
+      expect { subject.perform }.not_to change { RawUsageData.count }
+    end
+  end
+
+  context 'payload computation' do
+    it 'creates RawUsageData entry when there is NO entry with the same recorded_at timestamp' do
+      expect { subject.perform }.to change { RawUsageData.count }.by(1)
+    end
+
+    it 'updates RawUsageData entry when there is entry with the same recorded_at timestamp' do
+      record = create(:raw_usage_data, payload: { some_metric: 123 }, recorded_at: payload[:recorded_at])
+
+      expect { subject.perform }.to change { record.reload.payload }
+                                      .from("some_metric" => 123).to(payload.stringify_keys)
+    end
+
+    it 'reports errors and continue on execution' do
+      error = StandardError.new('some error')
+      allow(::ServicePing::BuildPayload).to receive(:new).and_raise(error)
+
+      expect(::Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(error)
+      expect_next_instance_of(::ServicePing::SubmitService, payload: nil) do |service|
+        expect(service).to receive(:execute)
+      end
+
+      subject.perform
+    end
   end
 
   it "obtains a #{described_class::LEASE_TIMEOUT} second exclusive lease" do