From c3b89d0b6375320549e821b866366ccf11551da2 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 15 Apr 2022 12:09:42 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../components/ci_variable_modal.vue | 2 +- app/models/onboarding_progress.rb | 3 +- .../_protected_paths.html.haml | 11 ++-- .../development/new_vulnerability_form.yml | 2 +- ...canning_action_to_onboarding_progresses.rb | 7 +++ db/schema_migrations/20220405061122 | 1 + db/structure.sql | 3 +- .../geo/replication/troubleshooting.md | 3 ++ doc/administration/gitaly/troubleshooting.md | 3 +- .../operations/fast_ssh_key_lookup.md | 3 +- .../troubleshooting/log_parsing.md | 27 +++++++--- .../vulnerability_report/index.md | 8 +-- lib/gitlab/diff/custom_diff.rb | 43 +++++++++++++-- .../gitlab_migration_large_project_spec.rb | 4 +- .../rubygems_registry_spec.rb | 3 +- .../components/ci_variable_modal_spec.js | 2 +- spec/lib/gitlab/diff/custom_diff_spec.rb | 53 +++++++++++++++++++ 17 files changed, 143 insertions(+), 35 deletions(-) create mode 100644 db/migrate/20220405061122_add_license_scanning_action_to_onboarding_progresses.rb create mode 100644 db/schema_migrations/20220405061122 diff --git a/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue b/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue index 2e198c59926..be2366108b3 100644 --- a/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue +++ b/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue @@ -405,7 +405,7 @@ export default { (actions) do diff --git a/app/views/admin/application_settings/_protected_paths.html.haml b/app/views/admin/application_settings/_protected_paths.html.haml index d273c81f51d..1f3f67c71c7 100644 --- a/app/views/admin/application_settings/_protected_paths.html.haml +++ b/app/views/admin/application_settings/_protected_paths.html.haml @@ -1,14 +1,11 @@ -= form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-protected-paths-settings'), html: { class: 'fieldset-form' } do |f| += gitlab_ui_form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-protected-paths-settings'), html: { class: 'fieldset-form' } do |f| = form_errors(@application_setting) %fieldset .form-group - .form-check - = f.check_box :throttle_protected_paths_enabled, class: 'form-check-input' - = f.label :throttle_protected_paths_enabled, class: 'form-check-label' do - = _('Enable rate limiting for POST requests to the specified paths') - %span.form-text.text-muted - = _('Helps reduce request volume for protected paths.') + = f.gitlab_ui_checkbox_component :throttle_protected_paths_enabled, + _('Enable rate limiting for POST requests to the specified paths'), + help_text: _('Helps reduce request volume for protected paths.') .form-group = f.label :throttle_protected_paths_requests_per_period, 'Maximum requests per period per user', class: 'label-bold' = f.number_field :throttle_protected_paths_requests_per_period, class: 'form-control gl-form-input' diff --git a/config/feature_flags/development/new_vulnerability_form.yml b/config/feature_flags/development/new_vulnerability_form.yml index ee812484928..8f73b18876c 100644 --- a/config/feature_flags/development/new_vulnerability_form.yml +++ b/config/feature_flags/development/new_vulnerability_form.yml @@ -5,4 +5,4 @@ rollout_issue_url: milestone: '14.9' type: development group: group::threat insights -default_enabled: false +default_enabled: true diff --git a/db/migrate/20220405061122_add_license_scanning_action_to_onboarding_progresses.rb b/db/migrate/20220405061122_add_license_scanning_action_to_onboarding_progresses.rb new file mode 100644 index 00000000000..0c07d1f18ba --- /dev/null +++ b/db/migrate/20220405061122_add_license_scanning_action_to_onboarding_progresses.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddLicenseScanningActionToOnboardingProgresses < Gitlab::Database::Migration[1.0] + def change + add_column :onboarding_progresses, :license_scanning_run_at, :datetime_with_timezone + end +end diff --git a/db/schema_migrations/20220405061122 b/db/schema_migrations/20220405061122 new file mode 100644 index 00000000000..c89ea9d692f --- /dev/null +++ b/db/schema_migrations/20220405061122 @@ -0,0 +1 @@ +2b20f2c3bb8dd5d3ba27dcb8854108763a40be9393f4799f16e9c10daf9fff75 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 771fce97e59..7a0d24df681 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -17697,7 +17697,8 @@ CREATE TABLE onboarding_progresses ( secure_secret_detection_run_at timestamp with time zone, secure_coverage_fuzzing_run_at timestamp with time zone, secure_cluster_image_scanning_run_at timestamp with time zone, - secure_api_fuzzing_run_at timestamp with time zone + secure_api_fuzzing_run_at timestamp with time zone, + license_scanning_run_at timestamp with time zone ); CREATE SEQUENCE onboarding_progresses_id_seq diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 03b7e0e4983..871d6041066 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -128,6 +128,9 @@ http://secondary.example.com/ Last status report was: 1 minute ago ``` +To find more details about failed items, check +[the `gitlab-rails/geo.log` file](../../troubleshooting/log_parsing.md#find-most-common-geo-sync-errors) + ### Check if PostgreSQL replication is working To check if PostgreSQL replication is working, check if: diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 9e783ad4e17..1be0bf23ed5 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -16,7 +16,8 @@ Before troubleshooting, see the Gitaly and Gitaly Cluster The following sections provide possible solutions to Gitaly errors. -See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings. +See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings, +and our advice on [parsing the `gitaly/current` file](../troubleshooting/log_parsing.md#parsing-gitalycurrent). ### Check versions when using standalone Gitaly servers diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 568d3e789bd..477a11dd899 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -31,7 +31,8 @@ able to accept a fingerprint. Check the version of OpenSSH on your server with ` ## Fast lookup is required for Geo **(PREMIUM)** -By default, GitLab manages an `authorized_keys` file that is located in the +Unlike [Cloud Native GitLab](https://docs.gitlab.com/charts/), Omnibus GitLab by default +manages an `authorized_keys` file that is located in the `git` user's home directory. For most installations, this will be located under `/var/opt/gitlab/.ssh/authorized_keys`, but you can use the following command to locate the `authorized_keys` on your system: diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index e4d5fa4843c..c5b1d302db2 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -11,6 +11,11 @@ but if they are not available you can still quickly parse [GitLab logs](../logs.md) in JSON format (the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/). +NOTE: +Spefically for summarising error events and basic usage statistics, +the GitLab Support Team provides the specialised +[`fast-stats` tool](https://gitlab.com/gitlab-com/support/toolbox/fast-stats/#when-to-use-it). + ## What is JQ? As noted in its [manual](https://stedolan.github.io/jq/manual/), `jq` is a command-line JSON processor. The following examples @@ -18,6 +23,10 @@ include use cases targeted for parsing GitLab log files. ## Parsing Logs +The examples listed below address their respective log files by +their relative Omnibus paths and default filenames. +Find the respective full paths in the [GitLab logs sections](../logs.md#production_jsonlog). + ### General Commands #### Pipe colorized `jq` output into `less` @@ -61,7 +70,7 @@ zcat some_json.log.25.gz | (head -1; tail -1) | jq '.time' grep -hR | jq -c -R 'fromjson?' | jq -C -s 'sort_by(.time)' | less -R ``` -### Parsing `production_json.log` and `api_json.log` +### Parsing `gitlab-rails/production_json.log` and `gitlab-rails/api_json.log` #### Find all requests with a 5XX status code @@ -111,7 +120,7 @@ jq 'select(.queue_duration > 10000)' jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; .[])' ``` -### Parsing `production_json.log` +### Parsing `gitlab-rails/production_json.log` #### Print the top three controller methods by request volume and their three longest durations @@ -127,7 +136,7 @@ CT: 2435 METHOD: MetricsController#index DURS: 299.29, 284.01, 158.57 CT: 1328 METHOD: Projects::NotesController#index DURS: 403.99, 386.29, 384.39 ``` -### Parsing `api_json.log` +### Parsing `gitlab-rails/api_json.log` #### Print top three routes with request count and their three longest durations @@ -157,6 +166,8 @@ jq --raw-output 'select(.severity == "ERROR") | [.project_path, .message] | @tsv ### Parsing `gitaly/current` +The following examples are useful to [troubleshoot Gitaly](../gitaly/troubleshooting.md). + #### Find all Gitaly requests sent from web UI ```shell @@ -197,7 +208,7 @@ jq --raw-output --slurp ' .[2]."grpc.time_ms", .[0]."grpc.request.glProjectPath" ] - | @sh' /var/log/gitlab/gitaly/current \ + | @sh' current \ | awk 'BEGIN { printf "%7s %10s %10s %10s\t%s\n", "CT", "MAX DURS", "", "", "PROJECT" } { printf "%7u %7u ms, %7u ms, %7u ms\t%s\n", $1, $2, $3, $4, $5 }' ``` @@ -215,12 +226,12 @@ jq --raw-output --slurp ' #### Find all projects affected by a fatal Git problem ```shell -grep "fatal: " /var/log/gitlab/gitaly/current | \ +grep "fatal: " current | \ jq '."grpc.request.glProjectPath"' | \ sort | uniq ``` -### Parsing `gitlab-shell.log` +### Parsing `gitlab-shell/gitlab-shell.log` For investigating Git calls via SSH, from [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/367). @@ -238,7 +249,7 @@ jq --raw-output --slurp ' | sort_by(-length) | limit(20; .[]) | "count: \(length)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ - /var/log/gitlab/gitlab-shell/gitlab-shell.log + gitlab-shell.log ``` Find the top 20 calls by project, user, and command: @@ -256,5 +267,5 @@ jq --raw-output --slurp ' | sort_by(-length) | limit(20; .[]) | "count: \(length)\tcommand: \(.[0].command)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ - /var/log/gitlab/gitlab-shell/gitlab-shell.log + gitlab-shell.log ``` diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index ea492b7fff8..a9cef15e3e8 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -222,12 +222,8 @@ To undo this action, select a different status from the same menu. ## Manually add a vulnerability finding -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301003) in GitLab 14.9. Disabled by default. - -FLAG: -This feature is not enabled by default. To make it available, ask an administrator to -[enable the feature flag](../../feature_flags.md) named `new_vulnerability_form`. -On GitLab.com, this feature is not yet available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301003) in GitLab 14.9. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/353796) in GitLab 14.10. To add a new vulnerability finding from your project level Vulnerability Report page: diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb index af1fd8fb03e..860f87a28a3 100644 --- a/lib/gitlab/diff/custom_diff.rb +++ b/lib/gitlab/diff/custom_diff.rb @@ -2,17 +2,29 @@ module Gitlab module Diff module CustomDiff + RENDERED_TIMEOUT_BACKGROUND = 20.seconds + RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds + BACKGROUND_EXECUTION = 'background' + FOREGROUND_EXECUTION = 'foreground' + LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' + LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' + LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + class << self def preprocess_before_diff(path, old_blob, new_blob) return unless path.ends_with? '.ipynb' - transformed_diff(old_blob&.data, new_blob&.data)&.tap do - transformed_for_diff(new_blob, old_blob) - Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + Timeout.timeout(timeout_time) do + transformed_diff(old_blob&.data, new_blob&.data)&.tap do + transformed_for_diff(new_blob, old_blob) + log_event(LOG_IPYNBDIFF_GENERATED) + end end + rescue Timeout::Error => e + rendered_timeout.increment(source: execution_source) + log_event(LOG_IPYNBDIFF_TIMEOUT, e) rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - Gitlab::ErrorTracking.log_exception(e) - nil + log_event(LOG_IPYNBDIFF_INVALID, e) end def transformed_diff(before, after) @@ -50,6 +62,27 @@ module Gitlab blobs_with_transformed_diffs[b] = true if b end end + + def rendered_timeout + @rendered_timeout ||= Gitlab::Metrics.counter( + :ipynb_semantic_diff_timeouts_total, + 'Counts the times notebook rendering timed out' + ) + end + + def timeout_time + Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND + end + + def execution_source + Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION + end + + def log_event(message, error = nil) + Gitlab::AppLogger.info({ message: message }) + Gitlab::ErrorTracking.track_exception(error) if error + nil + end end end end diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb index 8a7e07b82ab..9bf0955526f 100644 --- a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb @@ -2,7 +2,9 @@ # rubocop:disable Rails/Pluck, Layout/LineLength, RSpec/MultipleMemoizedHelpers module QA - RSpec.describe "Manage", :requires_admin, only: { job: 'large-gitlab-import' } do + RSpec.describe "Manage", requires_admin: 'uses admin API client for resource creation', + feature_flag: { name: 'bulk_import_projects', scope: :global }, + only: { job: 'large-gitlab-import' } do describe "Gitlab migration" do let(:logger) { Runtime::Logger.logger } let(:differ) { RSpec::Support::Differ.new(color: true) } diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb index 4a811363a83..409a1c10943 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/rubygems_registry_spec.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :orchestrated, :requires_admin, :packages, :object_storage do + RSpec.describe 'Package', :orchestrated, :packages, :object_storage, + feature_flag: { name: 'rubygem_packages', scope: :project } do describe 'RubyGems Repository' do include Runtime::Fixtures diff --git a/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js b/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js index 085ab1c0c30..2fedbbecd64 100644 --- a/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js +++ b/spec/frontend/ci_variable_list/components/ci_variable_modal_spec.js @@ -36,7 +36,7 @@ describe('Ci variable modal', () => { const findAddorUpdateButton = () => findModal() .findAll(GlButton) - .wrappers.find((button) => button.props('variant') === 'success'); + .wrappers.find((button) => button.props('variant') === 'confirm'); const deleteVariableButton = () => findModal() .findAll(GlButton) diff --git a/spec/lib/gitlab/diff/custom_diff_spec.rb b/spec/lib/gitlab/diff/custom_diff_spec.rb index 246508d2e1e..77d2a6cbcd6 100644 --- a/spec/lib/gitlab/diff/custom_diff_spec.rb +++ b/spec/lib/gitlab/diff/custom_diff_spec.rb @@ -34,6 +34,59 @@ RSpec.describe Gitlab::Diff::CustomDiff do expect(described_class.transformed_for_diff?(blob)).to be_falsey end end + + context 'timeout' do + subject { described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob) } + + it 'falls back to nil on timeout' do + allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + expect(Timeout).to receive(:timeout).and_raise(Timeout::Error) + + expect(subject).to be_nil + end + + context 'when in foreground' do + it 'utilizes timeout for web' do + expect(Timeout).to receive(:timeout).with(described_class::RENDERED_TIMEOUT_FOREGROUND).and_call_original + + expect(subject).not_to include('cells') + end + + it 'increments metrics' do + counter = Gitlab::Metrics.counter(:ipynb_semantic_diff_timeouts_total, 'desc') + + expect(Timeout).to receive(:timeout).and_raise(Timeout::Error) + expect { subject }.to change { counter.get(source: described_class::FOREGROUND_EXECUTION) }.by(1) + end + end + + context 'when in background' do + before do + allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true) + end + + it 'utilizes longer timeout for sidekiq' do + expect(Timeout).to receive(:timeout).with(described_class::RENDERED_TIMEOUT_BACKGROUND).and_call_original + + expect(subject).not_to include('cells') + end + + it 'increments metrics' do + counter = Gitlab::Metrics.counter(:ipynb_semantic_diff_timeouts_total, 'desc') + + expect(Timeout).to receive(:timeout).and_raise(Timeout::Error) + expect { subject }.to change { counter.get(source: described_class::BACKGROUND_EXECUTION) }.by(1) + end + end + end + + context 'when invalid ipynb' do + it 'returns nil' do + expect(ipynb_blob).to receive(:data).and_return('invalid ipynb') + + expect(described_class.preprocess_before_diff(ipynb_blob.path, nil, ipynb_blob)).to be_nil + end + end end describe '#transformed_blob_data' do