Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue

@@ -405,7 +405,7 @@ export default {
       <gl-button
         ref="updateOrAddVariable"
         :disabled="!canSubmit"
-        variant="success"
+        variant="confirm"
         category="primary"
         data-qa-selector="ci_variable_save_button"
         @click="updateOrAddVariable"

app/models/onboarding_progress.rb

@@ -27,7 +27,8 @@ class OnboardingProgress < ApplicationRecord
     :secure_secret_detection_run,
     :secure_coverage_fuzzing_run,
     :secure_api_fuzzing_run,
-    :secure_cluster_image_scanning_run
+    :secure_cluster_image_scanning_run,
+    :license_scanning_run
   ].freeze
 
   scope :incomplete_actions, -> (actions) do

app/views/admin/application_settings/_protected_paths.html.haml

@@ -1,14 +1,11 @@
-= form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-protected-paths-settings'), html: { class: 'fieldset-form' } do |f|
+= gitlab_ui_form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-protected-paths-settings'), html: { class: 'fieldset-form' } do |f|
   = form_errors(@application_setting)
 
   %fieldset
     .form-group
-      .form-check
-        = f.check_box :throttle_protected_paths_enabled, class: 'form-check-input'
-        = f.label :throttle_protected_paths_enabled, class: 'form-check-label' do
-          = _('Enable rate limiting for POST requests to the specified paths')
-        %span.form-text.text-muted
-          = _('Helps reduce request volume for protected paths.')
+      = f.gitlab_ui_checkbox_component :throttle_protected_paths_enabled,
+        _('Enable rate limiting for POST requests to the specified paths'),
+        help_text: _('Helps reduce request volume for protected paths.')
     .form-group
       = f.label :throttle_protected_paths_requests_per_period, 'Maximum requests per period per user', class: 'label-bold'
       = f.number_field :throttle_protected_paths_requests_per_period, class: 'form-control gl-form-input'

config/feature_flags/development/new_vulnerability_form.yml

@@ -5,4 +5,4 @@ rollout_issue_url:
 milestone: '14.9'
 type: development
 group: group::threat insights
-default_enabled: false
+default_enabled: true

db/migrate/20220405061122_add_license_scanning_action_to_onboarding_progresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLicenseScanningActionToOnboardingProgresses < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :onboarding_progresses, :license_scanning_run_at, :datetime_with_timezone
+  end
+end

db/schema_migrations/20220405061122

@@ -0,0 +1 @@
+2b20f2c3bb8dd5d3ba27dcb8854108763a40be9393f4799f16e9c10daf9fff75

db/structure.sql

@@ -17697,7 +17697,8 @@ CREATE TABLE onboarding_progresses (
     secure_secret_detection_run_at timestamp with time zone,
     secure_coverage_fuzzing_run_at timestamp with time zone,
     secure_cluster_image_scanning_run_at timestamp with time zone,
-    secure_api_fuzzing_run_at timestamp with time zone
+    secure_api_fuzzing_run_at timestamp with time zone,
+    license_scanning_run_at timestamp with time zone
 );
 
 CREATE SEQUENCE onboarding_progresses_id_seq

doc/administration/geo/replication/troubleshooting.md

@@ -128,6 +128,9 @@ http://secondary.example.com/
                 Last status report was: 1 minute ago
 ```
 
+To find more details about failed items, check
+[the `gitlab-rails/geo.log` file](../../troubleshooting/log_parsing.md#find-most-common-geo-sync-errors)
+
 ### Check if PostgreSQL replication is working
 
 To check if PostgreSQL replication is working, check if:

doc/administration/gitaly/troubleshooting.md

@@ -16,7 +16,8 @@ Before troubleshooting, see the Gitaly and Gitaly Cluster
 
 The following sections provide possible solutions to Gitaly errors.
 
-See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings.
+See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings,
+and our advice on [parsing the `gitaly/current` file](../troubleshooting/log_parsing.md#parsing-gitalycurrent).
 
 ### Check versions when using standalone Gitaly servers
 

doc/administration/operations/fast_ssh_key_lookup.md

@@ -31,7 +31,8 @@ able to accept a fingerprint. Check the version of OpenSSH on your server with `
 
 ## Fast lookup is required for Geo **(PREMIUM)**
 
-By default, GitLab manages an `authorized_keys` file that is located in the
+Unlike [Cloud Native GitLab](https://docs.gitlab.com/charts/), Omnibus GitLab by default
+manages an `authorized_keys` file that is located in the
 `git` user's home directory. For most installations, this will be located under
 `/var/opt/gitlab/.ssh/authorized_keys`, but you can use the following command to locate the `authorized_keys` on your system:
 

doc/administration/troubleshooting/log_parsing.md

@@ -11,6 +11,11 @@ but if they are not available you can still quickly parse
 [GitLab logs](../logs.md) in JSON format
 (the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/).
 
+NOTE:
+Spefically for summarising error events and basic usage statistics,
+the GitLab Support Team provides the specialised
+[`fast-stats` tool](https://gitlab.com/gitlab-com/support/toolbox/fast-stats/#when-to-use-it).
+
 ## What is JQ?
 
 As noted in its [manual](https://stedolan.github.io/jq/manual/), `jq` is a command-line JSON processor. The following examples
@@ -18,6 +23,10 @@ include use cases targeted for parsing GitLab log files.
 
 ## Parsing Logs
 
+The examples listed below address their respective log files by
+their relative Omnibus paths and default filenames.
+Find the respective full paths in the [GitLab logs sections](../logs.md#production_jsonlog).
+
 ### General Commands
 
 #### Pipe colorized `jq` output into `less`
@@ -61,7 +70,7 @@ zcat some_json.log.25.gz | (head -1; tail -1) | jq '.time'
 grep -hR <correlationID> | jq -c -R 'fromjson?' | jq -C -s 'sort_by(.time)'  | less -R
 ```
 
-### Parsing `production_json.log` and `api_json.log`
+### Parsing `gitlab-rails/production_json.log` and `gitlab-rails/api_json.log`
 
 #### Find all requests with a 5XX status code
 
@@ -111,7 +120,7 @@ jq 'select(.queue_duration > 10000)' <FILE>
 jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; .[])' <FILE>
 ```
 
-### Parsing `production_json.log`
+### Parsing `gitlab-rails/production_json.log`
 
 #### Print the top three controller methods by request volume and their three longest durations
 
@@ -127,7 +136,7 @@ CT: 2435   METHOD: MetricsController#index DURS: 299.29,  284.01,  158.57
 CT: 1328   METHOD: Projects::NotesController#index DURS: 403.99,  386.29,  384.39
 ```
 
-### Parsing `api_json.log`
+### Parsing `gitlab-rails/api_json.log`
 
 #### Print top three routes with request count and their three longest durations
 
@@ -157,6 +166,8 @@ jq --raw-output 'select(.severity == "ERROR") | [.project_path, .message] | @tsv
 
 ### Parsing `gitaly/current`
 
+The following examples are useful to [troubleshoot Gitaly](../gitaly/troubleshooting.md).
+
 #### Find all Gitaly requests sent from web UI
 
 ```shell
@@ -197,7 +208,7 @@ jq --raw-output --slurp '
       .[2]."grpc.time_ms",
       .[0]."grpc.request.glProjectPath"
     ]
-  | @sh' /var/log/gitlab/gitaly/current \
+  | @sh' current \
 | awk 'BEGIN { printf "%7s %10s %10s %10s\t%s\n", "CT", "MAX DURS", "", "", "PROJECT" }
   { printf "%7u %7u ms, %7u ms, %7u ms\t%s\n", $1, $2, $3, $4, $5 }'
 ```
@@ -215,12 +226,12 @@ jq --raw-output --slurp '
 #### Find all projects affected by a fatal Git problem
 
 ```shell
-grep "fatal: " /var/log/gitlab/gitaly/current | \
+grep "fatal: " current | \
     jq '."grpc.request.glProjectPath"' | \
     sort | uniq
 ```
 
-### Parsing `gitlab-shell.log`
+### Parsing `gitlab-shell/gitlab-shell.log`
 
 For investigating Git calls via SSH, from [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/367).
 
@@ -238,7 +249,7 @@ jq --raw-output --slurp '
   | sort_by(-length)
   | limit(20; .[])
   | "count: \(length)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \
-  /var/log/gitlab/gitlab-shell/gitlab-shell.log
+  gitlab-shell.log
 ```
 
 Find the top 20 calls by project, user, and command:
@@ -256,5 +267,5 @@ jq --raw-output --slurp '
   | sort_by(-length)
   | limit(20; .[])
   | "count: \(length)\tcommand: \(.[0].command)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \
-  /var/log/gitlab/gitlab-shell/gitlab-shell.log
+  gitlab-shell.log
 ```

doc/user/application_security/vulnerability_report/index.md

@@ -222,12 +222,8 @@ To undo this action, select a different status from the same menu.
 
 ## Manually add a vulnerability finding
 
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301003) in GitLab 14.9. Disabled by default.
-
-FLAG:
-This feature is not enabled by default. To make it available, ask an administrator to
-[enable the feature flag](../../feature_flags.md) named `new_vulnerability_form`.
-On GitLab.com, this feature is not yet available.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301003) in GitLab 14.9. Disabled by default.
+> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/353796) in GitLab 14.10.
 
 To add a new vulnerability finding from your project level Vulnerability Report page:
 

lib/gitlab/diff/custom_diff.rb

@@ -2,17 +2,29 @@
 module Gitlab
   module Diff
     module CustomDiff
+      RENDERED_TIMEOUT_BACKGROUND = 20.seconds
+      RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds
+      BACKGROUND_EXECUTION = 'background'
+      FOREGROUND_EXECUTION = 'foreground'
+      LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED'
+      LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT'
+      LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID'
+
       class << self
         def preprocess_before_diff(path, old_blob, new_blob)
           return unless path.ends_with? '.ipynb'
 
+          Timeout.timeout(timeout_time) do
             transformed_diff(old_blob&.data, new_blob&.data)&.tap do
               transformed_for_diff(new_blob, old_blob)
-            Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' })
+              log_event(LOG_IPYNBDIFF_GENERATED)
             end
+          end
+        rescue Timeout::Error => e
+          rendered_timeout.increment(source: execution_source)
+          log_event(LOG_IPYNBDIFF_TIMEOUT, e)
         rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e
-          Gitlab::ErrorTracking.log_exception(e)
-          nil
+          log_event(LOG_IPYNBDIFF_INVALID, e)
         end
 
         def transformed_diff(before, after)
@@ -50,6 +62,27 @@ module Gitlab
             blobs_with_transformed_diffs[b] = true if b
           end
         end
+
+        def rendered_timeout
+          @rendered_timeout ||= Gitlab::Metrics.counter(
+            :ipynb_semantic_diff_timeouts_total,
+            'Counts the times notebook rendering timed out'
+          )
+        end
+
+        def timeout_time
+          Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND
+        end
+
+        def execution_source
+          Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION
+        end
+
+        def log_event(message, error = nil)
+          Gitlab::AppLogger.info({ message: message })
+          Gitlab::ErrorTracking.track_exception(error) if error
+          nil
+        end
       end
     end
   end

qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb

@@ -2,7 +2,9 @@
 
 # rubocop:disable Rails/Pluck, Layout/LineLength, RSpec/MultipleMemoizedHelpers
 module QA
-  RSpec.describe "Manage", :requires_admin, only: { job: 'large-gitlab-import' } do
+  RSpec.describe "Manage", requires_admin: 'uses admin API client for resource creation',
+                           feature_flag: { name: 'bulk_import_projects', scope: :global },
+                           only: { job: 'large-gitlab-import' } do
     describe "Gitlab migration" do
       let(:logger) { Runtime::Logger.logger }
       let(:differ) { RSpec::Support::Differ.new(color: true) }

qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module QA
-  RSpec.describe 'Package', :orchestrated, :requires_admin, :packages, :object_storage do
+  RSpec.describe 'Package', :orchestrated, :packages, :object_storage,
+                            feature_flag: { name: 'rubygem_packages', scope: :project } do
     describe 'RubyGems Repository' do
       include Runtime::Fixtures
 

spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js

@@ -36,7 +36,7 @@ describe('Ci variable modal', () => {
   const findAddorUpdateButton = () =>
     findModal()
       .findAll(GlButton)
-      .wrappers.find((button) => button.props('variant') === 'success');
+      .wrappers.find((button) => button.props('variant') === 'confirm');
   const deleteVariableButton = () =>
     findModal()
       .findAll(GlButton)

spec/lib/gitlab/diff/custom_diff_spec.rb

@@ -34,6 +34,59 @@ RSpec.describe Gitlab::Diff::CustomDiff do
         expect(described_class.transformed_for_diff?(blob)).to be_falsey
       end
     end
+
+    context 'timeout' do
+      subject { described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob) }
+
+      it 'falls back to nil on timeout' do
+        allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception)
+        expect(Timeout).to receive(:timeout).and_raise(Timeout::Error)
+
+        expect(subject).to be_nil
+      end
+
+      context 'when in foreground' do
+        it 'utilizes timeout for web' do
+          expect(Timeout).to receive(:timeout).with(described_class::RENDERED_TIMEOUT_FOREGROUND).and_call_original
+
+          expect(subject).not_to include('cells')
+        end
+
+        it 'increments metrics' do
+          counter = Gitlab::Metrics.counter(:ipynb_semantic_diff_timeouts_total, 'desc')
+
+          expect(Timeout).to receive(:timeout).and_raise(Timeout::Error)
+          expect { subject }.to change { counter.get(source: described_class::FOREGROUND_EXECUTION) }.by(1)
+        end
+      end
+
+      context 'when in background' do
+        before do
+          allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
+        end
+
+        it 'utilizes longer timeout for sidekiq' do
+          expect(Timeout).to receive(:timeout).with(described_class::RENDERED_TIMEOUT_BACKGROUND).and_call_original
+
+          expect(subject).not_to include('cells')
+        end
+
+        it 'increments metrics' do
+          counter = Gitlab::Metrics.counter(:ipynb_semantic_diff_timeouts_total, 'desc')
+
+          expect(Timeout).to receive(:timeout).and_raise(Timeout::Error)
+          expect { subject }.to change { counter.get(source: described_class::BACKGROUND_EXECUTION) }.by(1)
+        end
+      end
+    end
+
+    context 'when invalid ipynb' do
+      it 'returns nil' do
+        expect(ipynb_blob).to receive(:data).and_return('invalid ipynb')
+
+        expect(described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob)).to be_nil
+      end
+    end
   end
 
   describe '#transformed_blob_data' do