Add latest changes from gitlab-org/gitlab@master

.rubocop_manual_todo.yml

@@ -1756,6 +1756,7 @@ Gitlab/NamespacedClass:
     - 'app/workers/git_garbage_collect_worker.rb'
     - 'app/workers/gitlab_performance_bar_stats_worker.rb'
     - 'app/workers/gitlab_shell_worker.rb'
+    - 'app/workers/gitlab_service_ping_worker.rb'
     - 'app/workers/gitlab_usage_ping_worker.rb'
     - 'app/workers/group_destroy_worker.rb'
     - 'app/workers/group_export_worker.rb'

app/assets/javascripts/packages_and_registries/infrastructure_registry/components/terraform_installation.vue

@@ -14,8 +14,7 @@ export default {
   computed: {
     ...mapState(['packageEntity', 'terraformHelpPath', 'gitlabHost', 'projectPath']),
     provisionInstructions() {
-      // eslint-disable-next-line @gitlab/require-i18n-strings
-      return `module "${this.packageEntity.name}" {
+      return `module "my_module_name" {
   source = "${this.gitlabHost}/${this.projectPath}/${this.packageEntity.name}"
   version = "${this.packageEntity.version}"
 }`;

app/workers/all_queues.yml

@@ -265,6 +265,15 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: cronjob:gitlab_service_ping
+  :worker_name: GitlabServicePingWorker
+  :feature_category: :service_ping
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent:
+  :tags: []
 - :name: cronjob:gitlab_usage_ping
   :worker_name: GitlabUsagePingWorker
   :feature_category: :service_ping

app/workers/gitlab_service_ping_worker.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class GitlabServicePingWorker # rubocop:disable Scalability/IdempotentWorker
+  LEASE_KEY = 'gitlab_service_ping_worker:ping'
+  LEASE_TIMEOUT = 86400
+
+  include ApplicationWorker
+  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+  include Gitlab::ExclusiveLeaseHelpers
+
+  feature_category :service_ping
+  sidekiq_options retry: 3, dead: false
+  sidekiq_retry_in { |count| (count + 1) * 8.hours.to_i }
+
+  def perform
+    # Disable service ping for GitLab.com
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/292929 for details
+    return if Gitlab.com?
+
+    # Multiple Sidekiq workers could run this. We should only do this at most once a day.
+    in_lock(LEASE_KEY, ttl: LEASE_TIMEOUT) do
+      # Splay the request over a minute to avoid thundering herd problems.
+      sleep(rand(0.0..60.0).round(3))
+
+      ServicePing::SubmitService.new.execute
+    end
+  end
+end

app/workers/gitlab_usage_ping_worker.rb

@@ -1,28 +1,4 @@
 # frozen_string_literal: true
 
-class GitlabUsagePingWorker # rubocop:disable Scalability/IdempotentWorker
-  LEASE_KEY = 'gitlab_usage_ping_worker:ping'
-  LEASE_TIMEOUT = 86400
-
-  include ApplicationWorker
-  include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
-  include Gitlab::ExclusiveLeaseHelpers
-
-  feature_category :service_ping
-  sidekiq_options retry: 3, dead: false
-  sidekiq_retry_in { |count| (count + 1) * 8.hours.to_i }
-
-  def perform
-    # Disable usage ping for GitLab.com
-    # See https://gitlab.com/gitlab-org/gitlab/-/issues/292929 for details
-    return if Gitlab.com?
-
-    # Multiple Sidekiq workers could run this. We should only do this at most once a day.
-    in_lock(LEASE_KEY, ttl: LEASE_TIMEOUT) do
-      # Splay the request over a minute to avoid thundering herd problems.
-      sleep(rand(0.0..60.0).round(3))
-
-      ServicePing::SubmitService.new.execute
-    end
-  end
+class GitlabUsagePingWorker < GitlabServicePingWorker # rubocop:disable Scalability/IdempotentWorker
 end

config/initializers/1_settings.rb

@@ -501,6 +501,9 @@ Settings.cron_jobs['stuck_export_jobs_worker']['job_class'] = 'StuckExportJobsWo
 Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= nil # This is dynamically loaded in the sidekiq initializer
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
+Settings.cron_jobs['gitlab_service_ping_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['gitlab_service_ping_worker']['cron'] ||= nil # This is dynamically loaded in the sidekiq initializer
+Settings.cron_jobs['gitlab_service_ping_worker']['job_class'] = 'GitlabServicePingWorker'
 Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
@@ -582,7 +585,7 @@ Settings.cron_jobs['batched_background_migrations_worker']['job_class'] = 'Datab
 
 Gitlab.ee do
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
-  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 0 1 * *'
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 1 * * *'
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
   Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'

config/settings.rb

@@ -163,7 +163,7 @@ class Settings < Settingslogic
     end
 
     def load_dynamic_cron_schedules!
-      cron_jobs['gitlab_usage_ping_worker']['cron'] ||= cron_for_usage_ping
+      cron_jobs['gitlab_service_ping_worker']['cron'] ||= cron_for_service_ping
     end
 
     private
@@ -197,7 +197,7 @@ class Settings < Settingslogic
     # Runs at a consistent random time of day on a day of the week based on
     # the instance UUID. This is to balance the load on the service receiving
     # these pings. The sidekiq job handles temporary http failures.
-    def cron_for_usage_ping
+    def cron_for_service_ping
       # Set a default UUID for the case when the UUID hasn't been initialized.
       uuid = Gitlab::CurrentSettings.uuid || 'uuid-not-set'
 

db/migrate/20210617180131_migrate_usage_ping_sidekiq_queue.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class MigrateUsagePingSidekiqQueue < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  # rubocop:disable Migration/SidekiqQueueMigrate
+  def up
+    sidekiq_queue_migrate 'cronjob:gitlab_usage_ping', to: 'cronjob:gitlab_service_ping'
+  end
+
+  def down
+    sidekiq_queue_migrate 'cronjob:gitlab_service_ping', to: 'cronjob:gitlab_usage_ping'
+  end
+  # rubocop:enable Migration/SidekiqQueueMigrate
+end

db/migrate/20210706084713_add_devops_adoption_snapshots_index.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddDevopsAdoptionSnapshotsIndex < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'idx_analytics_devops_adoption_snapshots_finalized'
+
+  def up
+    add_concurrent_index :analytics_devops_adoption_snapshots, [:namespace_id, :end_time], where: "recorded_at >= end_time", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :analytics_devops_adoption_snapshots, INDEX_NAME
+  end
+end

db/schema_migrations/20210617180131

@@ -0,0 +1 @@
+2adb38e71c6173350d1f98f3237b692e4f12c8a073115be23f3a713f69cde911

db/schema_migrations/20210706084713

@@ -0,0 +1 @@
+f69e3f50e4e5642a59e157a3c4a133090ec843b563e47198d560a54328176e56

db/structure.sql

@@ -22497,6 +22497,8 @@ CREATE UNIQUE INDEX i_ci_job_token_project_scope_links_on_source_and_target_proj
 
 CREATE INDEX idx_analytics_devops_adoption_segments_on_namespace_id ON analytics_devops_adoption_segments USING btree (namespace_id);
 
+CREATE INDEX idx_analytics_devops_adoption_snapshots_finalized ON analytics_devops_adoption_snapshots USING btree (namespace_id, end_time) WHERE (recorded_at >= end_time);
+
 CREATE INDEX idx_audit_events_part_on_entity_id_desc_author_id_created_at ON ONLY audit_events USING btree (entity_id, entity_type, id DESC, author_id, created_at);
 
 CREATE INDEX idx_award_emoji_on_user_emoji_name_awardable_type_awardable_id ON award_emoji USING btree (user_id, name, awardable_type, awardable_id);

doc/development/repository_mirroring.md

@@ -18,4 +18,34 @@ and the slides in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/86934
 Everything covered in this deep dive was accurate as of GitLab 11.6, and while specific
 details may have changed since then, it should still serve as a good introduction.
 
-<!-- vale gitlab.Spelling = YES -->
+<!-- vale gitlab.Spelling = YES -->
+
+## Explanation of mirroring process
+
+GitLab version 14 performs these steps when an
+[API call](../api/projects.md#start-the-pull-mirroring-process-for-a-project)
+triggers a pull mirror. Scheduled mirror updates are similar, but do not start with the API call:
+
+1. The request originates from an API call, and triggers the `start_pull_mirroring_service` in
+   [`project_mirror.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/project_mirror.rb).
+1. The pull mirroring service
+   ([`start_pull_mirroring_service.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/services/start_pull_mirroring_service.rb)) starts. It updates the project state, and forces the job to start immediately.
+1. The project import state is updated, and then triggers an `update_all_mirrors_worker` in
+   [`project_import_state.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/ee/project_import_state.rb#L170).
+1. The update all mirrors worker
+   ([`update_all_mirrors_worker.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/update_all_mirrors_worker.rb))
+   attempts to avoid stampedes by calling the `project_import_schedule` worker.
+1. The project import schedule worker
+   ([`project_import_schedule_worker.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/project_import_schedule_worker.rb#L21)) updates the state of the project, and
+   starts a Ruby `state_machine` to manage the import transition process.
+1. While updating the project state,
+   [this call in `project.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/ee/project.rb#L426)
+   starts the `repository_update_mirror` worker.
+1. The Sidekiq background mirror workers
+   ([`repository_update_mirror_worker.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/repository_update_mirror_worker.rb)) track the state of the mirroring task, and
+   provide good error state information. Processes can hang here, because this step manages the Git steps.
+1. The update mirror service
+   ([`update_mirror_service.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/services/projects/update_mirror_service.rb))
+   performs the Git operations.
+
+The import and mirror update processes are complete after the update mirror service step. However, depending on the changes included, more tasks (such as pipelines for commits) can be triggered.

doc/user/compliance/license_compliance/index.md

@@ -58,7 +58,7 @@ Java 8 and Gradle 1.x projects are not supported. The minimum supported version
 
 | Language   | Package managers                                                                             | Notes |
 |------------|----------------------------------------------------------------------------------------------|-------|
-| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/)                                    |       |
+| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) (7 and earlier)                    |       |
 | Go         | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) |       |
 | Java       | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/)                            |       |
 | .NET       | [NuGet](https://www.nuget.org/)                                                              | The .NET Framework is supported via the [mono project](https://www.mono-project.com/). There are, however, some limitations. The scanner doesn't support Windows-specific dependencies and doesn't report dependencies of your project's listed dependencies. Also, the scanner always marks detected licenses for all dependencies as `unknown`. |
@@ -74,12 +74,12 @@ The reported licenses might be incomplete or inaccurate.
 |------------|---------------------------------------------------------------------------------------------------------------|
 | JavaScript | [Yarn](https://yarnpkg.com/)                                                                                  |
 | Go         | `go get`, `gvt`, `glide`, `dep`, `trash`, `govendor`                                                          |
-| Erlang     | [Rebar](https://rebar3.org/)                                                                              |
+| Erlang     | [Rebar](https://rebar3.org/)                                                                                  |
 | Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage), [CocoaPods](https://cocoapods.org/) v0.39 and below |
 | Elixir     | [Mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html)                               |
 | C++/C      | [Conan](https://conan.io/)                                                                                    |
 | Scala      | [sbt](https://www.scala-sbt.org/)                                                                             |
-| Rust       | [Cargo](https://crates.io)                                                                                   |
+| Rust       | [Cargo](https://crates.io)                                                                                    |
 | PHP        | [Composer](https://getcomposer.org/)                                                                          |
 
 ## Requirements

doc/user/project/merge_requests/code_quality.md

@@ -379,7 +379,7 @@ After the Code Quality job completes:
 - The full list of code quality violations generated by a pipeline is shown in the
   Code Quality tab of the Pipeline Details page. **(PREMIUM)**
 
-### Generating an HTML report
+## Generate an HTML report
 
 In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/ci-cd/codequality/-/issues/10),
 it is possible to generate an HTML report file by setting the `REPORT_FORMAT`

lib/tasks/gitlab/usage_data.rake

@@ -19,7 +19,7 @@ namespace :gitlab do
 
     desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'
     task generate_and_send: :environment do
-      result = SubmitUsagePingService.new.execute
+      result = ServicePing::SubmitService.new.execute
 
       puts Gitlab::Json.pretty_generate(result.attributes)
     end

locale/gitlab.pot

@@ -6996,6 +6996,12 @@ msgstr ""
 msgid "ClusterAgents|Read more about getting started"
 msgstr ""
 
+msgid "ClusterAgents|Registering Agent"
+msgstr ""
+
+msgid "ClusterAgents|Select an Agent"
+msgstr ""
+
 msgid "ClusterAgents|The GitLab Agent also requires %{linkStart}enabling the Agent Server%{linkEnd}"
 msgstr ""
 

qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb

@@ -3,16 +3,18 @@
 module QA
   RSpec.describe 'Manage', :requires_admin do
     describe 'Bulk group import' do
-      let!(:admin_api_client) { Runtime::API::Client.as_admin }
-      let!(:user) do
+      let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') }
+
+      let(:admin_api_client) { Runtime::API::Client.as_admin }
+      let(:user) do
         Resource::User.fabricate_via_api! do |usr|
           usr.api_client = admin_api_client
           usr.hard_delete_on_api_removal = true
         end
       end
 
-      let!(:api_client) { Runtime::API::Client.new(user: user) }
-      let!(:personal_access_token) { api_client.personal_access_token }
+      let(:api_client) { Runtime::API::Client.new(user: user) }
+      let(:personal_access_token) { api_client.personal_access_token }
 
       let(:sandbox) do
         Resource::Sandbox.fabricate_via_api! do |group|
@@ -51,16 +53,10 @@ module QA
         end
       end
 
-      def staging?
-        Runtime::Scenario.gitlab_address.include?('staging.gitlab.com')
-      end
-
-      before(:all) do
+      before do
         Runtime::Feature.enable(:bulk_import) unless staging?
         Runtime::Feature.enable(:top_level_group_creation_enabled) if staging?
-      end
 
-      before do
         sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER)
 
         # create groups explicitly before connecting gitlab instance
@@ -116,9 +112,7 @@ module QA
 
       after do
         user.remove_via_api!
-      end
-
-      after(:all) do
+      ensure
         Runtime::Feature.disable(:bulk_import) unless staging?
         Runtime::Feature.disable(:top_level_group_creation_enabled) if staging?
       end

spec/config/settings_spec.rb

@@ -113,12 +113,12 @@ RSpec.describe Settings do
     end
   end
 
-  describe '.cron_for_usage_ping' do
+  describe '.cron_for_service_ping' do
     it 'returns correct crontab for some manually calculated example' do
       allow(Gitlab::CurrentSettings)
         .to receive(:uuid) { 'd9e2f4e8-db1f-4e51-b03d-f427e1965c4a'}
 
-      expect(described_class.send(:cron_for_usage_ping)).to eq('21 18 * * 4')
+      expect(described_class.send(:cron_for_service_ping)).to eq('21 18 * * 4')
     end
 
     it 'returns min, hour, day in the valid range' do
@@ -126,7 +126,7 @@ RSpec.describe Settings do
         .to receive(:uuid) { SecureRandom.uuid }
 
       10.times do
-        cron = described_class.send(:cron_for_usage_ping).split(/\s/)
+        cron = described_class.send(:cron_for_service_ping).split(/\s/)
 
         expect(cron[0].to_i).to be_between(0, 59)
         expect(cron[1].to_i).to be_between(0, 23)

spec/frontend/packages_and_registries/infrastructure_registry/components/__snapshots__/terraform_installation_spec.js.snap

@@ -10,7 +10,7 @@ exports[`TerraformInstallation renders all the messages 1`] = `
    
   <code-instruction-stub
     copytext="Copy Terraform Command"
-    instruction="module \\"Test/system-22\\" {
+    instruction="module \\"my_module_name\\" {
   source = \\"bar.dev/foo/Test/system-22\\"
   version = \\"0.1\\"
 }"

spec/frontend/packages_and_registries/infrastructure_registry/components/terraform_installation_spec.js

@@ -42,7 +42,7 @@ describe('TerraformInstallation', () => {
   describe('installation commands', () => {
     it('renders the correct command', () => {
       expect(findCodeInstructions().at(0).props('instruction')).toMatchInlineSnapshot(`
-        "module \\"Test/system-22\\" {
+        "module \\"my_module_name\\" {
           source = \\"bar.dev/foo/Test/system-22\\"
           version = \\"0.1\\"
         }"

spec/workers/every_sidekiq_worker_spec.rb

@@ -287,6 +287,7 @@ RSpec.describe 'Every Sidekiq worker' do
         'Gitlab::PhabricatorImport::ImportTasksWorker' => 5,
         'GitlabPerformanceBarStatsWorker' => 3,
         'GitlabShellWorker' => 3,
+        'GitlabServicePingWorker' => 3,
         'GitlabUsagePingWorker' => 3,
         'GroupDestroyWorker' => 3,
         'GroupExportWorker' => false,

spec/workers/gitlab_service_ping_worker_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe GitlabServicePingWorker, :clean_gitlab_redis_shared_state do
+  before do
+    allow_next_instance_of(ServicePing::SubmitService) { |service| allow(service).to receive(:execute) }
+    allow(subject).to receive(:sleep)
+  end
+
+  it 'does not run for GitLab.com' do
+    allow(Gitlab).to receive(:com?).and_return(true)
+    expect(ServicePing::SubmitService).not_to receive(:new)
+
+    subject.perform
+  end
+
+  it 'delegates to ServicePing::SubmitService' do
+    expect_next_instance_of(ServicePing::SubmitService) { |service| expect(service).to receive(:execute) }
+
+    subject.perform
+  end
+
+  it "obtains a #{described_class::LEASE_TIMEOUT} second exclusive lease" do
+    expect(Gitlab::ExclusiveLeaseHelpers::SleepingLock)
+      .to receive(:new)
+      .with(described_class::LEASE_KEY, hash_including(timeout: described_class::LEASE_TIMEOUT))
+      .and_call_original
+
+    subject.perform
+  end
+
+  it 'sleeps for between 0 and 60 seconds' do
+    expect(subject).to receive(:sleep).with(0..60)
+
+    subject.perform
+  end
+
+  context 'when lease is not obtained' do
+    before do
+      Gitlab::ExclusiveLease.new(described_class::LEASE_KEY, timeout: described_class::LEASE_TIMEOUT).try_obtain
+    end
+
+    it 'does not invoke ServicePing::SubmitService' do
+      allow_next_instance_of(ServicePing::SubmitService) { |service| expect(service).not_to receive(:execute) }
+
+      expect { subject.perform }.to raise_error(Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError)
+    end
+  end
+end