diff --git a/app/assets/javascripts/persistent_user_callouts.js b/app/assets/javascripts/persistent_user_callouts.js index 8c5f45e9d34..d4857a19ff7 100644 --- a/app/assets/javascripts/persistent_user_callouts.js +++ b/app/assets/javascripts/persistent_user_callouts.js @@ -7,6 +7,7 @@ const PERSISTENT_USER_CALLOUTS = [ '.js-buy-pipeline-minutes-notification-callout', '.js-token-expiry-callout', '.js-registration-enabled-callout', + '.js-new-user-signups-cap-reached', ]; const initCallouts = () => { diff --git a/app/controllers/admin/cohorts_controller.rb b/app/controllers/admin/cohorts_controller.rb index d5cd9c55422..a26dc554506 100644 --- a/app/controllers/admin/cohorts_controller.rb +++ b/app/controllers/admin/cohorts_controller.rb @@ -5,7 +5,7 @@ class Admin::CohortsController < Admin::ApplicationController track_unique_visits :index, target_id: 'i_analytics_cohorts' - feature_category :instance_statistics + feature_category :devops_reports def index if Gitlab::CurrentSettings.usage_ping_enabled diff --git a/app/controllers/admin/instance_review_controller.rb b/app/controllers/admin/instance_review_controller.rb index db304c82dd6..88ca2c88aab 100644 --- a/app/controllers/admin/instance_review_controller.rb +++ b/app/controllers/admin/instance_review_controller.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class Admin::InstanceReviewController < Admin::ApplicationController - feature_category :instance_statistics + feature_category :devops_reports def index redirect_to("#{::Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL}/instance_review?#{instance_review_params}") diff --git a/app/controllers/admin/instance_statistics_controller.rb b/app/controllers/admin/instance_statistics_controller.rb index 05a0a1ce314..30891fcfe7c 100644 --- a/app/controllers/admin/instance_statistics_controller.rb +++ b/app/controllers/admin/instance_statistics_controller.rb @@ -7,7 +7,7 @@ class Admin::InstanceStatisticsController < Admin::ApplicationController track_unique_visits :index, target_id: 'i_analytics_instance_statistics' - feature_category :instance_statistics + feature_category :devops_reports def index end diff --git a/app/controllers/concerns/wiki_actions.rb b/app/controllers/concerns/wiki_actions.rb index 6abb2e16226..65ab835c33c 100644 --- a/app/controllers/concerns/wiki_actions.rb +++ b/app/controllers/concerns/wiki_actions.rb @@ -8,6 +8,8 @@ module WikiActions include RedisTracking extend ActiveSupport::Concern + RESCUE_GIT_TIMEOUTS_IN = %w[show edit history diff pages].freeze + included do before_action { respond_to :html } @@ -38,6 +40,12 @@ module WikiActions feature: :track_unique_wiki_page_views, feature_default_enabled: true helper_method :view_file_button, :diff_file_html_data + + rescue_from ::Gitlab::Git::CommandTimedOut do |exc| + raise exc unless RESCUE_GIT_TIMEOUTS_IN.include?(action_name) + + render 'shared/wikis/git_error' + end end def new @@ -46,11 +54,7 @@ module WikiActions # rubocop:disable Gitlab/ModuleWithInstanceVariables def pages - @wiki_pages = Kaminari.paginate_array( - wiki.list_pages(sort: params[:sort], direction: params[:direction]) - ).page(params[:page]) - - @wiki_entries = WikiDirectory.group_pages(@wiki_pages) + @wiki_entries = WikiDirectory.group_pages(wiki_pages) render 'shared/wikis/pages' end @@ -225,9 +229,19 @@ module WikiActions unless @sidebar_page # Fallback to default sidebar @sidebar_wiki_entries, @sidebar_limited = wiki.sidebar_entries end + rescue ::Gitlab::Git::CommandTimedOut => e + @sidebar_error = e end # rubocop:enable Gitlab/ModuleWithInstanceVariables + def wiki_pages + strong_memoize(:wiki_pages) do + Kaminari.paginate_array( + wiki.list_pages(sort: params[:sort], direction: params[:direction]) + ).page(params[:page]) + end + end + def wiki_params params.require(:wiki).permit(:title, :content, :format, :message, :last_commit_sha) end diff --git a/app/controllers/projects/alert_management_controller.rb b/app/controllers/projects/alert_management_controller.rb index 8ecf8fadefd..ebe867d915d 100644 --- a/app/controllers/projects/alert_management_controller.rb +++ b/app/controllers/projects/alert_management_controller.rb @@ -3,7 +3,7 @@ class Projects::AlertManagementController < Projects::ApplicationController before_action :authorize_read_alert_management_alert! - feature_category :alert_management + feature_category :incident_management def index end diff --git a/app/controllers/projects/alerting/notifications_controller.rb b/app/controllers/projects/alerting/notifications_controller.rb index a3f4d784f25..86121bed381 100644 --- a/app/controllers/projects/alerting/notifications_controller.rb +++ b/app/controllers/projects/alerting/notifications_controller.rb @@ -10,7 +10,7 @@ module Projects prepend_before_action :repository, :project_without_auth - feature_category :alert_management + feature_category :incident_management def create token = extract_alert_manager_token(request) diff --git a/app/controllers/projects/prometheus/alerts_controller.rb b/app/controllers/projects/prometheus/alerts_controller.rb index 2892542e63c..7f711417f0b 100644 --- a/app/controllers/projects/prometheus/alerts_controller.rb +++ b/app/controllers/projects/prometheus/alerts_controller.rb @@ -16,7 +16,7 @@ module Projects before_action :authorize_read_prometheus_alerts!, except: [:notify] before_action :alert, only: [:update, :show, :destroy, :metrics_dashboard] - feature_category :alert_management + feature_category :incident_management def index render json: serialize_as_json(alerts) diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index cfad58fc0db..ad5651f9439 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -26,7 +26,8 @@ class UserCallout < ApplicationRecord suggest_pipeline: 22, customize_homepage: 23, feature_flags_new_version: 24, - registration_enabled_callout: 25 + registration_enabled_callout: 25, + new_user_signups_cap_reached: 26 # EE-only } validates :user, presence: true diff --git a/app/views/layouts/_page.html.haml b/app/views/layouts/_page.html.haml index f6fc49393d8..c552454caa7 100644 --- a/app/views/layouts/_page.html.haml +++ b/app/views/layouts/_page.html.haml @@ -17,6 +17,7 @@ = render_account_recovery_regular_check = render_if_exists "layouts/header/ee_subscribable_banner" = render_if_exists "shared/namespace_storage_limit_alert" + = render_if_exists "shared/new_user_signups_cap_reached_alert" = yield :customize_homepage_banner - unless @hide_breadcrumbs = render "layouts/nav/breadcrumbs" diff --git a/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml b/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml index 9d81fda68cb..549ca36cb6a 100644 --- a/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml +++ b/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml @@ -1,4 +1,4 @@ -- pretty_name = html_escape(@project&.full_name) || html_escape_once(_('<project name>')).html_safe +- pretty_name = @project&.full_name ? html_escape(@project&.full_name) : '<' + _('project name') + '>' - run_actions_text = html_escape(s_("ProjectService|Perform common operations on GitLab project: %{project_name}")) % { project_name: pretty_name } %p= s_("ProjectService|To set up this service:") diff --git a/app/views/projects/services/slack_slash_commands/_help.html.haml b/app/views/projects/services/slack_slash_commands/_help.html.haml index 86486d95eb7..67c43bd2f33 100644 --- a/app/views/projects/services/slack_slash_commands/_help.html.haml +++ b/app/views/projects/services/slack_slash_commands/_help.html.haml @@ -1,4 +1,4 @@ -- pretty_name = @project&.full_name || _('<project name>') +- pretty_name = @project&.full_name ? html_escape(@project&.full_name) : '<' + _('project name') + '>' - run_actions_text = html_escape_once(s_("ProjectService|Perform common operations on GitLab project: %{project_name}") % { project_name: pretty_name }) .info-well diff --git a/app/views/shared/_alert_info.html.haml b/app/views/shared/_alert_info.html.haml new file mode 100644 index 00000000000..e47c100909a --- /dev/null +++ b/app/views/shared/_alert_info.html.haml @@ -0,0 +1,6 @@ +.gl-alert.gl-alert-info + = sprite_icon('information-o', css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') + %button.js-close.gl-alert-dismiss{ type: 'button', 'aria-label' => _('Dismiss') } + = sprite_icon('close', css_class: 'gl-icon') + .gl-alert-body + = body diff --git a/app/views/shared/web_hooks/_form.html.haml b/app/views/shared/web_hooks/_form.html.haml index c5234f14090..4dd434ad1db 100644 --- a/app/views/shared/web_hooks/_form.html.haml +++ b/app/views/shared/web_hooks/_form.html.haml @@ -10,7 +10,7 @@ = s_('Webhooks|Use this token to validate received payloads. It will be sent with the request in the X-Gitlab-Token HTTP header.') .form-group = form.label :url, s_('Webhooks|Trigger'), class: 'label-bold' - %ul.list-unstyled.prepend-left-20 + %ul.list-unstyled.gl-ml-6 %li = form.check_box :push_events, class: 'form-check-input' = form.label :push_events, class: 'list-label form-check-label ml-1' do diff --git a/app/views/shared/wikis/_sidebar.html.haml b/app/views/shared/wikis/_sidebar.html.haml index c0ed7b4c6f2..7022762840e 100644 --- a/app/views/shared/wikis/_sidebar.html.haml +++ b/app/views/shared/wikis/_sidebar.html.haml @@ -10,11 +10,14 @@ = sprite_icon('download', css_class: 'gl-mr-2') %span= _("Clone repository") + - if @sidebar_error.present? + = render 'shared/alert_info', body: s_('Wiki|The sidebar failed to load. You can reload the page to try again.') + .blocks-container .block.block-first.w-100 - if @sidebar_page = render_wiki_content(@sidebar_page) - - else + - elsif @sidebar_wiki_entries %ul.wiki-pages = render @sidebar_wiki_entries, context: 'sidebar' .block.w-100 diff --git a/app/views/shared/wikis/git_error.html.haml b/app/views/shared/wikis/git_error.html.haml new file mode 100644 index 00000000000..dab3b940b9a --- /dev/null +++ b/app/views/shared/wikis/git_error.html.haml @@ -0,0 +1,14 @@ +- if @page + - wiki_page_title @page + +- add_page_specific_style 'page_bundles/wiki' + +- git_access_url = wiki_path(@wiki, action: :git_access) + +.wiki-page-header.top-area.gl-flex-direction-column.gl-lg-flex-direction-row + .gl-mt-5.gl-mb-3 + .gl-display-flex.gl-justify-content-space-between + %h2.gl-mt-0.gl-mb-5{ data: { qa_selector: 'wiki_page_title', testid: 'wiki_page_title' } }= @page ? @page.human_title : _('Failed to retrieve page') + .js-wiki-page-content.md.gl-pt-2{ data: { qa_selector: 'wiki_page_content', testid: 'wiki_page_content' } } + = _('The page could not be displayed because it timed out.') + = html_escape(_('You can view the source or %{linkStart}%{cloneIcon} clone the repository%{linkEnd}')) % { linkStart: "".html_safe, linkEnd: ''.html_safe, cloneIcon: sprite_icon('download', css_class: 'gl-mr-2').html_safe } diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 20adae5be5c..80204fd31b9 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -124,7 +124,7 @@ :idempotent: :tags: [] - :name: cronjob:analytics_instance_statistics_count_job_trigger - :feature_category: :instance_statistics + :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1329,7 +1329,7 @@ :idempotent: true :tags: [] - :name: analytics_instance_statistics_counter_job - :feature_category: :instance_statistics + :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb index bf57619fc6e..81a765d5d08 100644 --- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb +++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb @@ -8,7 +8,7 @@ module Analytics DEFAULT_DELAY = 3.minutes.freeze - feature_category :instance_statistics + feature_category :devops_reports urgency :low idempotent! diff --git a/app/workers/analytics/instance_statistics/counter_job_worker.rb b/app/workers/analytics/instance_statistics/counter_job_worker.rb index 7fc715419b8..c07b2569453 100644 --- a/app/workers/analytics/instance_statistics/counter_job_worker.rb +++ b/app/workers/analytics/instance_statistics/counter_job_worker.rb @@ -5,7 +5,7 @@ module Analytics class CounterJobWorker include ApplicationWorker - feature_category :instance_statistics + feature_category :devops_reports urgency :low idempotent! diff --git a/changelogs/unreleased/210345-update-index-adding-api-fuzzing.yml b/changelogs/unreleased/210345-update-index-adding-api-fuzzing.yml new file mode 100644 index 00000000000..8db9cb90493 --- /dev/null +++ b/changelogs/unreleased/210345-update-index-adding-api-fuzzing.yml @@ -0,0 +1,5 @@ +--- +title: Add index for API Fuzzing usage data +merge_request: 47692 +author: +type: performance diff --git a/changelogs/unreleased/233994_extend_usage_ping_api.yml b/changelogs/unreleased/233994_extend_usage_ping_api.yml new file mode 100644 index 00000000000..630e2ddbbd7 --- /dev/null +++ b/changelogs/unreleased/233994_extend_usage_ping_api.yml @@ -0,0 +1,5 @@ +--- +title: Add `increment_counter` to Usage Ping API +merge_request: 47309 +author: +type: added diff --git a/changelogs/unreleased/ajk-catch-wiki-timeouts.yml b/changelogs/unreleased/ajk-catch-wiki-timeouts.yml new file mode 100644 index 00000000000..05b38256b63 --- /dev/null +++ b/changelogs/unreleased/ajk-catch-wiki-timeouts.yml @@ -0,0 +1,5 @@ +--- +title: Catch wiki timeouts when rendering pages +merge_request: 46627 +author: +type: fixed diff --git a/config/feature_categories.yml b/config/feature_categories.yml index fb261377532..0f36e3e1727 100644 --- a/config/feature_categories.yml +++ b/config/feature_categories.yml @@ -9,7 +9,6 @@ --- - accessibility_testing - advanced_deployments -- alert_management - analysis - api - attack_emulation @@ -21,7 +20,7 @@ - boards - chatops - cloud_native_installation -- cluster_cost_optimization +- cluster_cost_management - code_analytics - code_quality - code_review @@ -47,6 +46,7 @@ - epics - error_tracking - feature_flags +- five_minute_production_app - foundations - fuzz_testing - gdk @@ -59,9 +59,10 @@ - helm_chart_registry - importers - incident_management +- infrastructure - infrastructure_as_code +- insider_threat - insights -- instance_statistics - integrations - interactive_application_security_testing - internationalization @@ -75,8 +76,11 @@ - load_testing - logging - malware_scanning +- memory - merge_trains - metrics +- mlops +- mobile_signing_deployment - navigation - omnibus_package - package_registry @@ -114,7 +118,7 @@ - tracing - usability_testing - users -- value_stream_analytics +- value_stream_management - vulnerability_database - vulnerability_management - web_firewall diff --git a/config/feature_flags/development/usage_data_static_site_editor_commits.yml b/config/feature_flags/development/usage_data_static_site_editor_commits.yml new file mode 100644 index 00000000000..a1d790b3505 --- /dev/null +++ b/config/feature_flags/development/usage_data_static_site_editor_commits.yml @@ -0,0 +1,8 @@ +--- +name: usage_data_static_site_editor_commits +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47309 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/284082 +milestone: '13.6' +type: development +group: group::static_site_editor +default_enabled: false diff --git a/config/feature_flags/development/usage_data_static_site_editor_merge_requests.yml b/config/feature_flags/development/usage_data_static_site_editor_merge_requests.yml new file mode 100644 index 00000000000..b68e4d12915 --- /dev/null +++ b/config/feature_flags/development/usage_data_static_site_editor_merge_requests.yml @@ -0,0 +1,8 @@ +--- +name: usage_data_static_site_editor_merge_requests +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47309 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/284083 +milestone: '13.6' +type: development +group: group::static_site_editor +default_enabled: false diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index cde7c7e3aed..ad1b3457c1a 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -288,6 +288,8 @@ - 1 - - repository_update_remote_mirror - 1 +- - requirements_management_import_requirements_csv + - 1 - - requirements_management_process_requirements_reports - 1 - - security_scans diff --git a/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb b/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb new file mode 100644 index 00000000000..9e8313f79f8 --- /dev/null +++ b/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class UpdateIndexSecureForApiFuzzingTelemetry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + OLD_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_created_at_parser_features' + NEW_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_name_created_at' + + def up + add_concurrent_index(:ci_builds, + [:user_id, :name, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('apifuzzer_fuzz'::character varying)::text, ('apifuzzer_fuzz_dnd'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: NEW_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, OLD_SECURE_INDEX_NAME + end + + def down + add_concurrent_index(:ci_builds, + [:user_id, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: OLD_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, NEW_SECURE_INDEX_NAME + end +end diff --git a/db/schema_migrations/20201113105000 b/db/schema_migrations/20201113105000 new file mode 100644 index 00000000000..0fe67898517 --- /dev/null +++ b/db/schema_migrations/20201113105000 @@ -0,0 +1 @@ +9f2c60df8e89f89d721f7f7917048eb914fa7c7726ec42dcb772ff7a90c54a9c \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 548238b2cea..1f76fbf430f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -21945,7 +21945,7 @@ CREATE UNIQUE INDEX index_scim_identities_on_user_id_and_group_id ON scim_identi CREATE UNIQUE INDEX index_scim_oauth_access_tokens_on_group_id_and_token_encrypted ON scim_oauth_access_tokens USING btree (group_id, token_encrypted); -CREATE INDEX index_secure_ci_builds_on_user_id_created_at_parser_features ON ci_builds USING btree (user_id, created_at) WHERE (((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('secret_detection'::character varying)::text]))); +CREATE INDEX index_secure_ci_builds_on_user_id_name_created_at ON ci_builds USING btree (user_id, name, created_at) WHERE (((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('apifuzzer_fuzz'::character varying)::text, ('apifuzzer_fuzz_dnd'::character varying)::text, ('secret_detection'::character varying)::text]))); CREATE INDEX index_security_ci_builds_on_name_and_id_parser_features ON ci_builds USING btree (name, id) WHERE (((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text)); diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 147d8407142..3fa743d70fb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -536,6 +536,15 @@ module API ) end + def increment_counter(event_name) + feature_name = "usage_data_#{event_name}" + return unless Feature.enabled?(feature_name) + + Gitlab::UsageDataCounters.count(event_name) + rescue => error + Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") + end + # @param event_name [String] the event name # @param values [Array|String] the values counted def increment_unique_values(event_name, values) diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index 1814e1a6782..6818c04fd2e 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -4,7 +4,7 @@ module API class Statistics < ::API::Base before { authenticated_as_admin! } - feature_category :instance_statistics + feature_category :devops_reports COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, MergeRequest, Note, Snippet, Key, Milestone].freeze diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 7b038ec74bb..cad2f52e951 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -20,6 +20,18 @@ module API requires :event, type: String, desc: 'The event name that should be tracked' end + post 'increment_counter' do + event_name = params[:event] + + increment_counter(event_name) + + status :ok + end + + params do + requires :event, type: String, desc: 'The event name that should be tracked' + end + post 'increment_unique_users' do event_name = params[:event] diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 96f3487fd6f..a2215366bdc 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -17,6 +17,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + CommandTimedOut = Class.new(CommandError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 9963bcfbf1c..2009683d32c 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -9,6 +9,8 @@ module Gitlab raise Gitlab::Git::Repository::NoRepository.new(e) rescue GRPC::InvalidArgument => e raise ArgumentError.new(e) + rescue GRPC::DeadlineExceeded => e + raise Gitlab::Git::CommandTimedOut.new(e) rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 4b0dd54683b..77a74c86c63 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -296,20 +296,7 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - [ - Gitlab::UsageDataCounters::WikiPageCounter, - Gitlab::UsageDataCounters::WebIdeCounter, - Gitlab::UsageDataCounters::NoteCounter, - Gitlab::UsageDataCounters::SnippetCounter, - Gitlab::UsageDataCounters::SearchCounter, - Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter, - Gitlab::UsageDataCounters::MergeRequestCounter, - Gitlab::UsageDataCounters::DesignsCounter, - Gitlab::UsageDataCounters::KubernetesAgentCounter, - Gitlab::UsageDataCounters::StaticSiteEditorCounter - ] + Gitlab::UsageDataCounters.counters end def components_usage_data diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb new file mode 100644 index 00000000000..79f0cd117bc --- /dev/null +++ b/lib/gitlab/usage_data_counters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + COUNTERS = [ + WikiPageCounter, + WebIdeCounter, + NoteCounter, + SnippetCounter, + SearchCounter, + CycleAnalyticsCounter, + ProductivityAnalyticsCounter, + SourceCodeCounter, + MergeRequestCounter, + DesignsCounter, + KubernetesAgentCounter, + StaticSiteEditorCounter + ].freeze + + UsageDataCounterError = Class.new(StandardError) + UnknownEvent = Class.new(UsageDataCounterError) + + class << self + def counters + self::COUNTERS + end + + def count(event_name) + counters.each do |counter| + event = counter.fetch_supported_event(event_name) + + return counter.count(event) if event + end + + raise UnknownEvent, "Cannot find counter for event #{event_name}" + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 44893645cc2..d28fd17a989 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -29,6 +29,12 @@ module Gitlab::UsageDataCounters known_events.map { |event| [counter_key(event), -1] }.to_h end + def fetch_supported_event(event_name) + return if prefix.present? && !event_name.start_with?(prefix) + + known_events.find { |event| counter_key(event) == event_name.to_sym } + end + private def require_known_event(event) diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb index 61f98887adc..46aec52b95a 100644 --- a/lib/gitlab/usage_data_counters/search_counter.rb +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -4,6 +4,7 @@ module Gitlab module UsageDataCounters class SearchCounter < BaseCounter KNOWN_EVENTS = %w[all_searches navbar_searches].freeze + PREFIX = nil class << self def redis_key(event) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 40b02230f0e..1006fa3c25a 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -906,9 +906,6 @@ msgstr "" msgid "<no scopes selected>" msgstr "" -msgid "<project name>" -msgstr "" - msgid "'%{data}' at %{location} does not match format: %{format}" msgstr "" @@ -2272,6 +2269,15 @@ msgstr "" msgid "Administration" msgstr "" +msgid "Admin|Additional users must be reviewed and approved by a system administrator. Learn more about %{help_link_start}usage caps%{help_link_end}." +msgstr "" + +msgid "Admin|View pending user approvals" +msgstr "" + +msgid "Admin|Your instance has reached its user cap" +msgstr "" + msgid "Advanced" msgstr "" @@ -11444,6 +11450,9 @@ msgstr "" msgid "Failed to reset key. Please try again." msgstr "" +msgid "Failed to retrieve page" +msgstr "" + msgid "Failed to save merge conflicts resolutions. Please try again!" msgstr "" @@ -27028,6 +27037,9 @@ msgstr "" msgid "The number of times an upload record could not find its file" msgstr "" +msgid "The page could not be displayed because it timed out." +msgstr "" + msgid "The parent epic is confidential and can only contain confidential epics and issues" msgstr "" @@ -30707,6 +30719,9 @@ msgstr "" msgid "Wiki|Pages" msgstr "" +msgid "Wiki|The sidebar failed to load. You can reload the page to try again." +msgstr "" + msgid "Wiki|Title" msgstr "" @@ -31007,6 +31022,9 @@ msgstr "" msgid "You can test your .gitlab-ci.yml in %{linkStart}CI Lint%{linkEnd}." msgstr "" +msgid "You can view the source or %{linkStart}%{cloneIcon} clone the repository%{linkEnd}" +msgstr "" + msgid "You cannot access the raw file. Please wait a minute." msgstr "" @@ -32777,6 +32795,9 @@ msgstr "" msgid "project members" msgstr "" +msgid "project name" +msgstr "" + msgid "projects" msgstr "" diff --git a/rubocop/rubocop-usage-data.yml b/rubocop/rubocop-usage-data.yml index 51ad3ed0bef..bff58cb46b2 100644 --- a/rubocop/rubocop-usage-data.yml +++ b/rubocop/rubocop-usage-data.yml @@ -20,6 +20,7 @@ UsageData/LargeTable: - :Gitlab::Runtime - :Gitaly::Server - :Gitlab::UsageData + - :Gitlab::UsageDataCounters - :License - :Rails - :Time diff --git a/spec/lib/gitlab/usage_data_counters/base_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/base_counter_spec.rb new file mode 100644 index 00000000000..4a31191d75f --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/base_counter_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::UsageDataCounters::BaseCounter do + describe '.fetch_supported_event' do + subject { described_class.fetch_supported_event(event_name) } + + let(:event_name) { 'generic_event' } + let(:prefix) { 'generic' } + let(:known_events) { %w[event another_event] } + + before do + allow(described_class).to receive(:prefix) { prefix } + allow(described_class).to receive(:known_events) { known_events } + end + + it 'returns the matching event' do + is_expected.to eq 'event' + end + + context 'when event is unknown' do + let(:event_name) { 'generic_unknown_event' } + + it { is_expected.to be_nil } + end + + context 'when prefix does not match the event name' do + let(:prefix) { 'special' } + + it { is_expected.to be_nil } + end + end +end diff --git a/spec/lib/gitlab/usage_data_counters/search_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/search_counter_spec.rb index b55e20ba555..17188a75ccb 100644 --- a/spec/lib/gitlab/usage_data_counters/search_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/search_counter_spec.rb @@ -20,4 +20,12 @@ RSpec.describe Gitlab::UsageDataCounters::SearchCounter, :clean_gitlab_redis_sha context 'navbar_searches counter' do it_behaves_like 'usage counter with totals', :navbar_searches end + + describe '.fetch_supported_event' do + subject { described_class.fetch_supported_event(event_name) } + + let(:event_name) { 'all_searches' } + + it { is_expected.to eq 'all_searches' } + end end diff --git a/spec/lib/gitlab/usage_data_counters_spec.rb b/spec/lib/gitlab/usage_data_counters_spec.rb new file mode 100644 index 00000000000..379a2cb778d --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::UsageDataCounters do + describe '.usage_data_counters' do + subject { described_class.counters } + + it { is_expected.to all(respond_to :totals) } + it { is_expected.to all(respond_to :fallback_totals) } + end + + describe '.count' do + subject { described_class.count(event_name) } + + let(:event_name) { 'static_site_editor_views' } + + it 'increases a view counter' do + expect(Gitlab::UsageDataCounters::StaticSiteEditorCounter).to receive(:count).with('views') + + subject + end + + context 'when event_name is not defined' do + let(:event_name) { 'unknown' } + + it 'raises an exception' do + expect { subject }.to raise_error(Gitlab::UsageDataCounters::UnknownEvent) + end + end + end +end diff --git a/spec/requests/api/usage_data_spec.rb b/spec/requests/api/usage_data_spec.rb index 4f4f386e9db..d44f179eed8 100644 --- a/spec/requests/api/usage_data_spec.rb +++ b/spec/requests/api/usage_data_spec.rb @@ -5,6 +5,87 @@ require 'spec_helper' RSpec.describe API::UsageData do let_it_be(:user) { create(:user) } + describe 'POST /usage_data/increment_counter' do + let(:endpoint) { '/usage_data/increment_counter' } + let(:known_event) { "#{known_event_prefix}_#{known_event_postfix}" } + let(:known_event_prefix) { "static_site_editor" } + let(:known_event_postfix) { 'commits' } + let(:unknown_event) { 'unknown' } + + context 'without CSRF token' do + it 'returns forbidden' do + stub_feature_flags(usage_data_api: true) + allow(Gitlab::RequestForgeryProtection).to receive(:verified?).and_return(false) + + post api(endpoint, user), params: { event: known_event } + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + + context 'usage_data_api feature not enabled' do + it 'returns not_found' do + stub_feature_flags(usage_data_api: false) + + post api(endpoint, user), params: { event: known_event } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'without authentication' do + it 'returns 401 response' do + post api(endpoint), params: { event: known_event } + + expect(response).to have_gitlab_http_status(:unauthorized) + end + end + + context 'with authentication' do + before do + stub_feature_flags(usage_data_api: true) + stub_feature_flags("usage_data_#{known_event}" => true) + stub_application_setting(usage_ping_enabled: true) + allow(Gitlab::RequestForgeryProtection).to receive(:verified?).and_return(true) + end + + context 'when event is missing from params' do + it 'returns bad request' do + post api(endpoint, user), params: {} + + expect(response).to have_gitlab_http_status(:bad_request) + end + end + + %w[merge_requests commits].each do |postfix| + context 'with correct params' do + let(:known_event_postfix) { postfix } + + it 'returns status ok' do + expect(Gitlab::UsageDataCounters::BaseCounter).to receive(:count).with(known_event_postfix) + post api(endpoint, user), params: { event: known_event } + + expect(response).to have_gitlab_http_status(:ok) + end + end + end + + context 'with unknown event' do + before do + skip_feature_flags_yaml_validation + end + + it 'returns status ok' do + expect(Gitlab::UsageDataCounters::BaseCounter).not_to receive(:count) + + post api(endpoint, user), params: { event: unknown_event } + + expect(response).to have_gitlab_http_status(:ok) + end + end + end + end + describe 'POST /usage_data/increment_unique_users' do let(:endpoint) { '/usage_data/increment_unique_users' } let(:known_event) { 'g_compliance_dashboard' } diff --git a/spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb b/spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb index a6ad8fc594c..4c8a45d3b90 100644 --- a/spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb +++ b/spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb @@ -14,6 +14,22 @@ RSpec.shared_examples 'wiki controller actions' do sign_in(user) end + shared_examples 'recovers from git timeout' do + let(:method_name) { :page } + + context 'when we encounter git command errors' do + it 'renders the appropriate template', :aggregate_failures do + expect(controller).to receive(method_name) do + raise ::Gitlab::Git::CommandTimedOut, 'Deadline Exceeded' + end + + request + + expect(response).to render_template('shared/wikis/git_error') + end + end + end + describe 'GET #new' do subject(:request) { get :new, params: routing_params } @@ -48,6 +64,12 @@ RSpec.shared_examples 'wiki controller actions' do get :pages, params: routing_params.merge(id: wiki_title) end + it_behaves_like 'recovers from git timeout' do + subject(:request) { get :pages, params: routing_params.merge(id: wiki_title) } + + let(:method_name) { :wiki_pages } + end + it 'assigns the page collections' do expect(assigns(:wiki_pages)).to contain_exactly(an_instance_of(WikiPage)) expect(assigns(:wiki_entries)).to contain_exactly(an_instance_of(WikiPage)) @@ -99,6 +121,12 @@ RSpec.shared_examples 'wiki controller actions' do end end + it_behaves_like 'recovers from git timeout' do + subject(:request) { get :history, params: routing_params.merge(id: wiki_title) } + + let(:allow_read_wiki) { true } + end + it_behaves_like 'fetching history', :ok do let(:allow_read_wiki) { true } @@ -139,6 +167,10 @@ RSpec.shared_examples 'wiki controller actions' do expect(response).to have_gitlab_http_status(:not_found) end end + + it_behaves_like 'recovers from git timeout' do + subject(:request) { get :diff, params: routing_params.merge(id: wiki_title, version_id: wiki.repository.commit.id) } + end end describe 'GET #show' do @@ -151,6 +183,8 @@ RSpec.shared_examples 'wiki controller actions' do context 'when page exists' do let(:id) { wiki_title } + it_behaves_like 'recovers from git timeout' + it 'renders the page' do request @@ -161,6 +195,28 @@ RSpec.shared_examples 'wiki controller actions' do expect(assigns(:sidebar_limited)).to be(false) end + context 'the sidebar fails to load' do + before do + allow(Wiki).to receive(:for_container).and_return(wiki) + wiki.wiki + expect(wiki).to receive(:find_sidebar) do + raise ::Gitlab::Git::CommandTimedOut, 'Deadline Exceeded' + end + end + + it 'renders the page, and marks the sidebar as failed' do + request + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to render_template('shared/wikis/_sidebar') + expect(assigns(:page).title).to eq(wiki_title) + expect(assigns(:sidebar_page)).to be_nil + expect(assigns(:sidebar_wiki_entries)).to be_nil + expect(assigns(:sidebar_limited)).to be_nil + expect(assigns(:sidebar_error)).to be_a_kind_of(::Gitlab::Git::CommandError) + end + end + context 'page view tracking' do it_behaves_like 'tracking unique hll events', :track_unique_wiki_page_views do let(:target_id) { 'wiki_action' } @@ -308,6 +364,7 @@ RSpec.shared_examples 'wiki controller actions' do subject(:request) { get(:edit, params: routing_params.merge(id: id_param)) } it_behaves_like 'edit action' + it_behaves_like 'recovers from git timeout' context 'when page content encoding is valid' do render_views diff --git a/spec/views/shared/wikis/_sidebar.html.haml_spec.rb b/spec/views/shared/wikis/_sidebar.html.haml_spec.rb new file mode 100644 index 00000000000..ddf223efd99 --- /dev/null +++ b/spec/views/shared/wikis/_sidebar.html.haml_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe 'shared/wikis/_sidebar.html.haml' do + let_it_be(:project) { create(:project) } + let_it_be(:wiki) { Wiki.for_container(project, project.default_owner) } + + before do + assign(:wiki, wiki) + assign(:project, project) + end + + it 'includes a link to clone the repository' do + render + + expect(rendered).to have_link('Clone repository') + end + + context 'the wiki is not a project wiki' do + it 'does not include the clone repository link' do + allow(wiki).to receive(:container).and_return(create(:group)) + + render + + expect(rendered).not_to have_link('Clone repository') + end + end + + context 'the sidebar failed to load' do + before do + assign(:sidebar_error, Object.new) + end + + it 'reports this to the user' do + render + + expect(rendered).to include('The sidebar failed to load') + expect(rendered).to have_css('.gl-alert.gl-alert-info') + end + end + + context 'The sidebar comes from a custom page' do + before do + assign(:sidebar_page, double('WikiPage', path: 'sidebar.md', slug: 'sidebar', content: 'Some sidebar content')) + end + + it 'does not show an alert' do + render + + expect(rendered).not_to include('The sidebar failed to load') + expect(rendered).not_to have_css('.gl-alert.gl-alert-info') + end + + it 'renders the wiki content' do + render + + expect(rendered).to include('Some sidebar content') + end + end + + context 'The sidebar comes a list of wiki pages' do + before do + assign(:sidebar_wiki_entries, create_list(:wiki_page, 3, wiki: wiki)) + assign(:sidebar_limited, true) + stub_template "../shared/wikis/_wiki_pages.html.erb" => "Entries: <%= @sidebar_wiki_entries.size %>" + stub_template "../shared/wikis/_wiki_page.html.erb" => 'A WIKI PAGE' + end + + it 'does not show an alert' do + render + + expect(rendered).not_to include('The sidebar failed to load') + expect(rendered).not_to have_css('.gl-alert.gl-alert-info') + end + + it 'renders the wiki content' do + render + + expect(rendered).to include('A WIKI PAGE' * 3) + expect(rendered).to have_link('View All Pages') + end + + context 'there is no more to see' do + it 'does not invite the user to view more' do + assign(:sidebar_limited, false) + + render + + expect(rendered).not_to have_link('View All Pages') + end + end + end +end