Add latest changes from gitlab-org/gitlab@master

config/feature_flags/development/use_redis_hll_instrumentation_classes.yml

@@ -0,0 +1,8 @@
+---
+name: use_redis_hll_instrumentation_classes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90237 
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/365332
+milestone: '15.1'
+type: development
+group: group::product intelligence
+default_enabled: false

doc/administration/invalidate_markdown_cache.md

@@ -5,18 +5,25 @@ info: "To determine the technical writer assigned to the Stage/Group associated
 type: reference
 ---
 
-# Invalidate Markdown Cache **(FREE)**
+# Markdown cache **(FREE)**
 
-For performance reasons, GitLab caches the HTML version of Markdown text
-in fields like comments, issue descriptions, and merge request descriptions. These
-cached versions can become outdated, such as
-when the `external_url` configuration option is changed. Links
+For performance reasons, GitLab caches the HTML version of Markdown text in fields such as:
+
+- Comments.
+- Issue descriptions.
+- Merge request descriptions.
+
+These cached versions can become outdated, such as when the `external_url` configuration option is changed. Links
 in the cached text would still refer to the old URL.
 
-To avoid this problem, the administrator can invalidate the existing cache by
-increasing the `local_markdown_version` setting in application settings. This can
-be done by changing the application settings
-[through the API](../api/settings.md#change-application-settings):
+## Invalidate the cache
+
+Pre-requisite:
+
+- You must be an administrator.
+
+To avoid problems caused by cached HTML versions, invalidate the existing cache by increasing the `local_markdown_version`
+setting in application settings [using the API](../api/settings.md#change-application-settings):
 
 ```shell
 curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/settings?local_markdown_version=<increased_number>"

doc/ci/runners/index.md

@@ -12,7 +12,7 @@ No configuration is required. Your jobs can run on:
 
 - [Linux runners](saas/linux_saas_runner.md).
 - [Windows runners](saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)).
-- [macOS runners](saas/macos_saas_runner.md) ([Limited Availability](../../policy/alpha-beta-support.md#limited-availability-la)).
+- [macOS runners](saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)).
 
 The number of minutes you can use on these runners depends on the
 [maximum number of CI/CD minutes](../pipelines/cicd_minutes.md)

doc/user/infrastructure/iac/troubleshooting.md

@@ -112,3 +112,12 @@ job that returned the error:
 
 1. Configure the [CI/CD environment scope](../../../ci/variables/#add-a-cicd-variable-to-a-project) for the job.
 1. Set the job's [environment](../../../ci/yaml/#environment), matching the environment scope from the previous step.
+
+### Error refreshing state: HTTP remote state endpoint requires auth 
+
+To resolve this, ensure that:
+
+- The access token you use has `api` scope.
+- If you have set the `TF_HTTP_PASSWORD` CI/CD variable, make sure that you either:
+  - Set the same value as `TF_PASSWORD`
+  - Remove `TF_HTTP_PASSWORD` variable if your CI/CD job does not explicitly use it.

lib/gitlab/usage_data_counters/hll_redis_counter.rb

@@ -37,6 +37,10 @@ module Gitlab
         user_packages
       ].freeze
 
+      CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[
+        error_tracking
+      ].freeze
+
       # Track event on entity_id
       # Increment a Redis HLL counter for unique event_name and entity_id
       #
@@ -114,7 +118,7 @@ module Gitlab
         #   - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store.
         #   - The aggregation used doesn't affect data granulation.
         def unique_events_data
-          categories.each_with_object({}) do |category, category_results|
+          categories_pending_migration.each_with_object({}) do |category, category_results|
             events_names = events_for_category(category)
 
             event_results = events_names.each_with_object({}) do |event, hash|
@@ -148,6 +152,14 @@ module Gitlab
 
         private
 
+        def categories_pending_migration
+          if ::Feature.enabled?(:use_redis_hll_instrumentation_classes)
+            (categories - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS)
+          else
+            categories
+          end
+        end
+
         def track(values, event_name, context: '', time: Time.zone.now)
           return unless ::ServicePing::ServicePingSettings.enabled?
 

qa/qa/specs/features/browser_ui/1_manage/user/follow_user_activity_spec.rb

@@ -24,6 +24,7 @@ module QA
       let(:group) do
         group = QA::Resource::Group.fabricate_via_api! do |group|
           group.path = "group_for_follow_user_activity_#{SecureRandom.hex(8)}"
+          group.api_client = admin_api_client
         end
         group.add_member(followed_user, Resource::Members::AccessLevel::MAINTAINER)
         group

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -21,6 +21,81 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
     travel_to(reference_time) { example.run }
   end
 
+  context 'migration to instrumentation classes data collection' do
+    let_it_be(:instrumented_events) do
+      ::Gitlab::Usage::MetricDefinition.all.map do |definition|
+        next unless definition.attributes[:instrumentation_class] == 'RedisHLLMetric' && definition.available?
+
+        definition.attributes.dig(:options, :events)&.sort
+      end.compact.to_set
+    end
+
+    def not_instrumented_events(category)
+      described_class
+        .events_for_category(category)
+        .sort
+        .reject do |event|
+          instrumented_events.include?([event])
+        end
+    end
+
+    def not_instrumented_aggregate(category)
+      events = described_class.events_for_category(category).sort
+
+      return unless described_class::CATEGORIES_FOR_TOTALS.include?(category)
+      return unless described_class.send(:eligible_for_totals?, events)
+      return if instrumented_events.include?(events)
+
+      events
+    end
+
+    describe 'Gitlab::UsageDataCounters::HLLRedisCounter::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS' do
+      it 'includes only fully migrated categories' do
+        wrong_skipped_events = described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS.map do |category|
+          next if not_instrumented_events(category).empty? && not_instrumented_aggregate(category).nil?
+
+          [category, [not_instrumented_events(category), not_instrumented_aggregate(category)].compact]
+        end.compact.to_h
+
+        expect(wrong_skipped_events).to be_empty
+      end
+
+      context 'with not instrumented category' do
+        let(:instrumented_events) { [] }
+
+        it 'can detect not migrated category' do
+          wrong_skipped_events = described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS.map do |category|
+            next if not_instrumented_events(category).empty? && not_instrumented_aggregate(category).nil?
+
+            [category, [not_instrumented_events(category), not_instrumented_aggregate(category)].compact]
+          end.compact.to_h
+
+          expect(wrong_skipped_events).not_to be_empty
+        end
+      end
+    end
+
+    describe '.unique_events_data' do
+      context 'with use_redis_hll_instrumentation_classes feature enabled' do
+        it 'does not include instrumented categories' do
+          stub_feature_flags(use_redis_hll_instrumentation_classes: true)
+
+          expect(described_class.unique_events_data.keys)
+            .not_to include(*described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS)
+        end
+      end
+
+      context 'with use_redis_hll_instrumentation_classes feature disabled' do
+        it 'includes instrumented categories' do
+          stub_feature_flags(use_redis_hll_instrumentation_classes: false)
+
+          expect(described_class.unique_events_data.keys)
+            .to include(*described_class::CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS)
+        end
+      end
+    end
+  end
+
   describe '.categories' do
     it 'gets all unique category names' do
       expect(described_class.categories).to contain_exactly(

spec/lib/gitlab/usage_data_spec.rb

@@ -1212,6 +1212,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
     let(:ignored_metrics) { ["i_package_composer_deploy_token_weekly"] }
 
     it 'has all known_events' do
+      stub_feature_flags(use_redis_hll_instrumentation_classes: false)
       expect(subject).to have_key(:redis_hll_counters)
 
       expect(subject[:redis_hll_counters].keys).to match_array(categories)