diff --git a/app/models/analytics/cycle_analytics/stage_event_hash.rb b/app/models/analytics/cycle_analytics/stage_event_hash.rb new file mode 100644 index 00000000000..0e1e9b3ef67 --- /dev/null +++ b/app/models/analytics/cycle_analytics/stage_event_hash.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class StageEventHash < ApplicationRecord + has_many :cycle_analytics_project_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage', inverse_of: :stage_event_hash + + validates :hash_sha256, presence: true + + # Creates or queries the id of the corresponding stage event hash code + def self.record_id_by_hash_sha256(hash) + casted_hash_code = Arel::Nodes.build_quoted(hash, Analytics::CycleAnalytics::StageEventHash.arel_table[:hash_sha256]).to_sql + + # Atomic, safe insert without retrying + query = <<~SQL + WITH insert_cte AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + INSERT INTO #{quoted_table_name} (hash_sha256) VALUES (#{casted_hash_code}) ON CONFLICT DO NOTHING RETURNING ID + ) + SELECT ids.id FROM ( + (SELECT id FROM #{quoted_table_name} WHERE hash_sha256=#{casted_hash_code} LIMIT 1) + UNION ALL + (SELECT id FROM insert_cte LIMIT 1) + ) AS ids LIMIT 1 + SQL + + connection.execute(query).first['id'] + end + + def self.cleanup_if_unused(id) + unused_hashes_for(id) + .where(id: id) + .delete_all + end + + def self.unused_hashes_for(id) + exists_query = Analytics::CycleAnalytics::ProjectStage.where(stage_event_hash_id: id).select('1').limit(1) + where.not('EXISTS (?)', exists_query) + end + end + end +end +Analytics::CycleAnalytics::StageEventHash.prepend_mod_with('Analytics::CycleAnalytics::StageEventHash') diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 2a0274f5706..7bb6004ca83 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -10,6 +10,7 @@ module Analytics included do belongs_to :start_event_label, class_name: 'GroupLabel', optional: true belongs_to :end_event_label, class_name: 'GroupLabel', optional: true + belongs_to :stage_event_hash, class_name: 'Analytics::CycleAnalytics::StageEventHash', foreign_key: :stage_event_hash_id, optional: true validates :name, presence: true validates :name, exclusion: { in: Gitlab::Analytics::CycleAnalytics::DefaultStages.names }, if: :custom? @@ -28,6 +29,9 @@ module Analytics scope :ordered, -> { order(:relative_position, :id) } scope :for_list, -> { includes(:start_event_label, :end_event_label).ordered } scope :by_value_stream, -> (value_stream) { where(value_stream_id: value_stream.id) } + + before_save :ensure_stage_event_hash_id + after_commit :cleanup_old_stage_event_hash end def parent=(_) @@ -133,6 +137,20 @@ module Analytics .id_in(label_id) .exists? end + + def ensure_stage_event_hash_id + previous_stage_event_hash = stage_event_hash&.hash_sha256 + + if previous_stage_event_hash.blank? || events_hash_code != previous_stage_event_hash + self.stage_event_hash_id = Analytics::CycleAnalytics::StageEventHash.record_id_by_hash_sha256(events_hash_code) + end + end + + def cleanup_old_stage_event_hash + if stage_event_hash_id_previously_changed? && stage_event_hash_id_previously_was + Analytics::CycleAnalytics::StageEventHash.cleanup_if_unused(stage_event_hash_id_previously_was) + end + end end end end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 526ccae36b0..54b11ea6041 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -337,7 +337,7 @@ class ProjectPolicy < BasePolicy enable :read_metrics_user_starred_dashboard end - rule { packages_disabled | repository_disabled }.policy do + rule { packages_disabled }.policy do prevent(*create_read_update_admin_destroy(:package)) end diff --git a/app/views/admin/application_settings/_signin.html.haml b/app/views/admin/application_settings/_signin.html.haml index 12a9f949750..156e7d3fb76 100644 --- a/app/views/admin/application_settings/_signin.html.haml +++ b/app/views/admin/application_settings/_signin.html.haml @@ -6,22 +6,22 @@ .form-check = f.check_box :password_authentication_enabled_for_web, class: 'form-check-input' = f.label :password_authentication_enabled_for_web, class: 'form-check-label' do - = _('Password authentication enabled for web interface') + = _('Allow password authentication for the web interface') .form-text.text-muted - = _('When disabled, an external authentication provider must be used.') + = _('When inactive, an external authentication provider must be used.') .form-group .form-check = f.check_box :password_authentication_enabled_for_git, class: 'form-check-input' = f.label :password_authentication_enabled_for_git, class: 'form-check-label' do - = _('Password authentication enabled for Git over HTTP(S)') + = _('Allow password authentication for Git over HTTP(S)') .form-text.text-muted - When disabled, a Personal Access Token + When inactive, a Personal Access Token - if Gitlab::Auth::Ldap::Config.enabled? or LDAP password must be used to authenticate. - if omniauth_enabled? && button_based_providers.any? %fieldset.form-group - %legend.gl-font-base.gl-mb-3.gl-border-none.gl-font-weight-bold= _('Enabled OAuth sign-in sources') + %legend.gl-font-base.gl-mb-3.gl-border-none.gl-font-weight-bold= _('Enabled OAuth authentication sources') = hidden_field_tag 'application_setting[enabled_oauth_sign_in_sources][]' - oauth_providers_checkboxes.each do |source| = source @@ -30,39 +30,44 @@ .form-check = f.check_box :require_two_factor_authentication, class: 'form-check-input' = f.label :require_two_factor_authentication, class: 'form-check-label' do - = _('Require all users to set up two-factor authentication') + = _('Enforce two-factor authentication') + %p.form-text.text-muted + = _('Enforce two-factor authentication for all user sign-ins.') + = link_to _('Learn more.'), help_page_path('security/two_factor_authentication.md'), target: '_blank', rel: 'noopener noreferrer' + .form-group + = f.label :two_factor_authentication, _('Two-factor grace period'), class: 'label-bold' + = f.number_field :two_factor_grace_period, min: 0, class: 'form-control gl-form-input', placeholder: '0' + .form-text.text-muted + = _('Maximum time that users are allowed to skip the setup of two-factor authentication (in hours). Set to 0 (zero) to enforce at next sign in.') .form-group = f.label :admin_mode, _('Admin Mode'), class: 'label-bold' = sprite_icon('lock', css_class: 'gl-icon') .form-check = f.check_box :admin_mode, class: 'form-check-input' = f.label :admin_mode, class: 'form-check-label' do - = _('Require additional authentication for administrative tasks') - .form-text.text-muted - = link_to _('Learn more.'), help_page_path('user/admin_area/settings/sign_in_restrictions', anchor: 'admin-mode') + = _('Enable admin mode') + %p.form-text.text-muted + = _('Require additional authentication for administrative tasks.') + = link_to _('Learn more.'), help_page_path('user/admin_area/settings/sign_in_restrictions', anchor: 'admin-mode'), target: '_blank', rel: 'noopener noreferrer' .form-group = f.label :unknown_sign_in, _('Email notification for unknown sign-ins'), class: 'label-bold' .form-check = f.check_box :notify_on_unknown_sign_in, class: 'form-check-input' = f.label :notify_on_unknown_sign_in, class: 'form-check-label' do - = _('Notify users by email when sign-in location is not recognized') - = link_to sprite_icon('question-o'), - 'https://docs.gitlab.com/ee/user/profile/unknown_sign_in_notification.html', - target: '_blank' - .form-group - = f.label :two_factor_authentication, _('Two-factor grace period (hours)'), class: 'label-bold' - = f.number_field :two_factor_grace_period, min: 0, class: 'form-control gl-form-input', placeholder: '0' - .form-text.text-muted= _('Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication') + = _('Enable email notification') + %p.form-text.text-muted + = _('Notify users by email when sign-in location is not recognized.') + = link_to _('Learn more.'), help_page_path('user/profile/unknown_sign_in_notification.md'), target: '_blank', rel: 'noopener noreferrer' .form-group = f.label :home_page_url, _('Home page URL'), class: 'label-bold' = f.text_field :home_page_url, class: 'form-control gl-form-input', placeholder: 'http://company.example.com', :'aria-describedby' => 'home_help_block' - %span.form-text.text-muted#home_help_block= _("We will redirect non-logged in users to this page") + %span.form-text.text-muted#home_help_block= _("Direct non-authenticated users to this page.") .form-group - = f.label :after_sign_out_path, _('After sign-out path'), class: 'label-bold' + = f.label :after_sign_out_path, _('Sign-out page URL'), class: 'label-bold' = f.text_field :after_sign_out_path, class: 'form-control gl-form-input', placeholder: 'http://company.example.com', :'aria-describedby' => 'after_sign_out_path_help_block' - %span.form-text.text-muted#after_sign_out_path_help_block= _("We will redirect users to this page after they sign out") + %span.form-text.text-muted#home_help_block= _("Direct users to this page after they sign out.") .form-group = f.label :sign_in_text, _('Sign-in text'), class: 'label-bold' = f.text_area :sign_in_text, class: 'form-control gl-form-input', rows: 4 - .form-text.text-muted Markdown enabled + %span.form-text.text-muted#home_help_block= _("Add text to the sign-in page. Markdown enabled.") = f.submit _('Save changes'), class: "gl-button btn btn-confirm" diff --git a/app/views/admin/application_settings/general.html.haml b/app/views/admin/application_settings/general.html.haml index 595c4fc08c0..9102769cc6e 100644 --- a/app/views/admin/application_settings/general.html.haml +++ b/app/views/admin/application_settings/general.html.haml @@ -53,7 +53,8 @@ %button.btn.gl-button.btn-default.js-settings-toggle{ type: 'button' } = expanded_by_default? ? _('Collapse') : _('Expand') %p - = _('Set requirements for a user to sign-in. Enable mandatory two-factor authentication.') + = _('Set sign-in restrictions for all users.') + = link_to _('Learn more.'), help_page_path('user/admin_area/settings/sign_in_restrictions.md'), target: '_blank', rel: 'noopener noreferrer' .settings-content = render 'signin' diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index a59fa2bd624..5b4567dde29 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -14,9 +14,10 @@ class WebHookWorker worker_has_external_dependencies! def perform(hook_id, data, hook_name) - hook = WebHook.find(hook_id) - data = data.with_indifferent_access + hook = WebHook.find_by_id(hook_id) + return unless hook + data = data.with_indifferent_access WebHookService.new(hook, data, hook_name, jid).execute end end diff --git a/config/application.rb b/config/application.rb index 4c9c4711c66..e6608ddbc20 100644 --- a/config/application.rb +++ b/config/application.rb @@ -168,6 +168,11 @@ module Gitlab # like if you have constraints or database-specific column types config.active_record.schema_format = :sql + # Dump all DB schemas even if schema_search_path is defined, + # so that we get the same db/structure.sql + # regardless if schema_search_path is set, or not. + config.active_record.dump_schemas = :all + # Use new connection handling so that we can use Rails 6.1+ multiple # database support. config.active_record.legacy_connection_handling = false diff --git a/config/feature_flags/development/get_tag_signatures.yml b/config/feature_flags/development/get_tag_signatures.yml new file mode 100644 index 00000000000..e0d7d5d6dcf --- /dev/null +++ b/config/feature_flags/development/get_tag_signatures.yml @@ -0,0 +1,8 @@ +--- +name: get_tag_signatures +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67000 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337842 +milestone: '14.2' +type: development +group: group::gitaly +default_enabled: false diff --git a/config/metrics/counts_28d/20210216184259_p_terraform_state_api_unique_users_monthly.yml b/config/metrics/counts_28d/20210216184259_p_terraform_state_api_unique_users_monthly.yml index 2aec3aaa14c..2cec39f73ce 100644 --- a/config/metrics/counts_28d/20210216184259_p_terraform_state_api_unique_users_monthly.yml +++ b/config/metrics/counts_28d/20210216184259_p_terraform_state_api_unique_users_monthly.yml @@ -10,6 +10,10 @@ value_type: number status: data_available time_frame: 28d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - p_terraform_state_api_unique_users distribution: - ce - ee diff --git a/config/metrics/counts_28d/20210216184303_o_pipeline_authoring_unique_users_committing_ciconfigfile_monthly.yml b/config/metrics/counts_28d/20210216184303_o_pipeline_authoring_unique_users_committing_ciconfigfile_monthly.yml index ec5b2045f1c..babaaae2225 100644 --- a/config/metrics/counts_28d/20210216184303_o_pipeline_authoring_unique_users_committing_ciconfigfile_monthly.yml +++ b/config/metrics/counts_28d/20210216184303_o_pipeline_authoring_unique_users_committing_ciconfigfile_monthly.yml @@ -10,6 +10,10 @@ value_type: number status: data_available time_frame: 28d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_committing_ciconfigfile distribution: - ee - ce diff --git a/config/metrics/counts_28d/20210301144228_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_monthly.yml b/config/metrics/counts_28d/20210301144228_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_monthly.yml index 680061913d1..7c3ba5fc1dc 100644 --- a/config/metrics/counts_28d/20210301144228_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_monthly.yml +++ b/config/metrics/counts_28d/20210301144228_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_monthly.yml @@ -12,6 +12,10 @@ milestone: "13.10" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54707 time_frame: 28d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile distribution: - ce - ee diff --git a/config/metrics/counts_28d/20210409095855_users_expanding_secure_security_report_monthly.yml b/config/metrics/counts_28d/20210409095855_users_expanding_secure_security_report_monthly.yml index 58f8c709d3f..bae8dd4d544 100644 --- a/config/metrics/counts_28d/20210409095855_users_expanding_secure_security_report_monthly.yml +++ b/config/metrics/counts_28d/20210409095855_users_expanding_secure_security_report_monthly.yml @@ -12,6 +12,10 @@ milestone: '13.11' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57133 time_frame: 28d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - users_expanding_secure_security_report distribution: - ce - ee diff --git a/config/metrics/counts_28d/20210427105033_pipeline_authoring_total_unique_counts_monthly.yml b/config/metrics/counts_28d/20210427105033_pipeline_authoring_total_unique_counts_monthly.yml index 2d2b9fd4fe8..03f4ed9ce85 100644 --- a/config/metrics/counts_28d/20210427105033_pipeline_authoring_total_unique_counts_monthly.yml +++ b/config/metrics/counts_28d/20210427105033_pipeline_authoring_total_unique_counts_monthly.yml @@ -12,7 +12,12 @@ status: data_available milestone: "13.12" introduced_by_url: time_frame: 28d -data_source: +data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_committing_ciconfigfile + - o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile distribution: - ce - ee diff --git a/config/metrics/counts_7d/20210216184301_o_pipeline_authoring_unique_users_committing_ciconfigfile_weekly.yml b/config/metrics/counts_7d/20210216184301_o_pipeline_authoring_unique_users_committing_ciconfigfile_weekly.yml index f032e8a8558..1a81aabe9ac 100644 --- a/config/metrics/counts_7d/20210216184301_o_pipeline_authoring_unique_users_committing_ciconfigfile_weekly.yml +++ b/config/metrics/counts_7d/20210216184301_o_pipeline_authoring_unique_users_committing_ciconfigfile_weekly.yml @@ -10,6 +10,10 @@ value_type: number status: data_available time_frame: 7d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_committing_ciconfigfile distribution: - ee - ce diff --git a/config/metrics/counts_7d/20210301144209_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_weekly.yml b/config/metrics/counts_7d/20210301144209_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_weekly.yml index c0fe97815d9..d89ad057a55 100644 --- a/config/metrics/counts_7d/20210301144209_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_weekly.yml +++ b/config/metrics/counts_7d/20210301144209_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_weekly.yml @@ -12,6 +12,10 @@ milestone: "13.10" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54707 time_frame: 7d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile distribution: - ce - ee diff --git a/config/metrics/counts_7d/20210409095855_users_expanding_secure_security_report_weekly.yml b/config/metrics/counts_7d/20210409095855_users_expanding_secure_security_report_weekly.yml index f4ade05ed51..f6de8e608e3 100644 --- a/config/metrics/counts_7d/20210409095855_users_expanding_secure_security_report_weekly.yml +++ b/config/metrics/counts_7d/20210409095855_users_expanding_secure_security_report_weekly.yml @@ -12,6 +12,10 @@ milestone: '13.11' introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57133 time_frame: 7d data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - users_expanding_secure_security_report distribution: - ce - ee diff --git a/config/metrics/counts_7d/20210427105030_pipeline_authoring_total_unique_counts_weekly.yml b/config/metrics/counts_7d/20210427105030_pipeline_authoring_total_unique_counts_weekly.yml index 259bc4f4357..85cc6f63db1 100644 --- a/config/metrics/counts_7d/20210427105030_pipeline_authoring_total_unique_counts_weekly.yml +++ b/config/metrics/counts_7d/20210427105030_pipeline_authoring_total_unique_counts_weekly.yml @@ -12,7 +12,12 @@ status: data_available milestone: "13.12" introduced_by_url: time_frame: 7d -data_source: +data_source: redis_hll +instrumentation_class: RedisHLLMetric +options: + events: + - o_pipeline_authoring_unique_users_committing_ciconfigfile + - o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile distribution: - ce - ee diff --git a/db/migrate/20210730101609_create_analytics_cycle_analytics_stage_event_hashes.rb b/db/migrate/20210730101609_create_analytics_cycle_analytics_stage_event_hashes.rb new file mode 100644 index 00000000000..ad517fbbff0 --- /dev/null +++ b/db/migrate/20210730101609_create_analytics_cycle_analytics_stage_event_hashes.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CreateAnalyticsCycleAnalyticsStageEventHashes < ActiveRecord::Migration[6.1] + def change + create_table :analytics_cycle_analytics_stage_event_hashes do |t| + t.binary :hash_sha256 + t.index :hash_sha256, unique: true, name: 'index_cycle_analytics_stage_event_hashes_on_hash_sha_256' + end + end +end diff --git a/db/migrate/20210730102952_add_stage_hash_fk_to_project_stages.rb b/db/migrate/20210730102952_add_stage_hash_fk_to_project_stages.rb new file mode 100644 index 00000000000..2909df78a6a --- /dev/null +++ b/db/migrate/20210730102952_add_stage_hash_fk_to_project_stages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddStageHashFkToProjectStages < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + unless column_exists?(:analytics_cycle_analytics_project_stages, :stage_event_hash_id) + add_column :analytics_cycle_analytics_project_stages, :stage_event_hash_id, :bigint + end + + add_concurrent_index :analytics_cycle_analytics_project_stages, :stage_event_hash_id, name: 'index_project_stages_on_stage_event_hash_id' + add_concurrent_foreign_key :analytics_cycle_analytics_project_stages, :analytics_cycle_analytics_stage_event_hashes, column: :stage_event_hash_id, on_delete: :cascade + end + + def down + remove_column :analytics_cycle_analytics_project_stages, :stage_event_hash_id + end +end diff --git a/db/migrate/20210730103808_add_stage_hash_fk_to_group_stages.rb b/db/migrate/20210730103808_add_stage_hash_fk_to_group_stages.rb new file mode 100644 index 00000000000..3cd53f2bb50 --- /dev/null +++ b/db/migrate/20210730103808_add_stage_hash_fk_to_group_stages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddStageHashFkToGroupStages < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + unless column_exists?(:analytics_cycle_analytics_group_stages, :stage_event_hash_id) + add_column :analytics_cycle_analytics_group_stages, :stage_event_hash_id, :bigint + end + + add_concurrent_index :analytics_cycle_analytics_group_stages, :stage_event_hash_id, name: 'index_group_stages_on_stage_event_hash_id' + add_concurrent_foreign_key :analytics_cycle_analytics_group_stages, :analytics_cycle_analytics_stage_event_hashes, column: :stage_event_hash_id, on_delete: :cascade + end + + def down + remove_column :analytics_cycle_analytics_group_stages, :stage_event_hash_id + end +end diff --git a/db/schema_migrations/20210730101609 b/db/schema_migrations/20210730101609 new file mode 100644 index 00000000000..f3522b194b2 --- /dev/null +++ b/db/schema_migrations/20210730101609 @@ -0,0 +1 @@ +f819eaed7e387f18f066180cbf9d0849b3e38db95bbf3e8487d3bc58d9b489ae \ No newline at end of file diff --git a/db/schema_migrations/20210730102952 b/db/schema_migrations/20210730102952 new file mode 100644 index 00000000000..a21008671c2 --- /dev/null +++ b/db/schema_migrations/20210730102952 @@ -0,0 +1 @@ +cb97b869bfb0b76dd0684aca1f40c86e7c1c9c9a0d52684830115288088e8066 \ No newline at end of file diff --git a/db/schema_migrations/20210730103808 b/db/schema_migrations/20210730103808 new file mode 100644 index 00000000000..6467a981cbb --- /dev/null +++ b/db/schema_migrations/20210730103808 @@ -0,0 +1 @@ +5c104ffdb64943aa4828a9b961c8f9141dfd2ae861cea7116722d2b0d4598957 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index ad92c14adec..935c81eb530 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9085,7 +9085,8 @@ CREATE TABLE analytics_cycle_analytics_group_stages ( hidden boolean DEFAULT false NOT NULL, custom boolean DEFAULT true NOT NULL, name character varying(255) NOT NULL, - group_value_stream_id bigint NOT NULL + group_value_stream_id bigint NOT NULL, + stage_event_hash_id bigint ); CREATE SEQUENCE analytics_cycle_analytics_group_stages_id_seq @@ -9128,7 +9129,8 @@ CREATE TABLE analytics_cycle_analytics_project_stages ( hidden boolean DEFAULT false NOT NULL, custom boolean DEFAULT true NOT NULL, name character varying(255) NOT NULL, - project_value_stream_id bigint NOT NULL + project_value_stream_id bigint NOT NULL, + stage_event_hash_id bigint ); CREATE SEQUENCE analytics_cycle_analytics_project_stages_id_seq @@ -9158,6 +9160,20 @@ CREATE SEQUENCE analytics_cycle_analytics_project_value_streams_id_seq ALTER SEQUENCE analytics_cycle_analytics_project_value_streams_id_seq OWNED BY analytics_cycle_analytics_project_value_streams.id; +CREATE TABLE analytics_cycle_analytics_stage_event_hashes ( + id bigint NOT NULL, + hash_sha256 bytea +); + +CREATE SEQUENCE analytics_cycle_analytics_stage_event_hashes_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE analytics_cycle_analytics_stage_event_hashes_id_seq OWNED BY analytics_cycle_analytics_stage_event_hashes.id; + CREATE TABLE analytics_devops_adoption_segments ( id bigint NOT NULL, last_recorded_at timestamp with time zone, @@ -19925,6 +19941,8 @@ ALTER TABLE ONLY analytics_cycle_analytics_project_stages ALTER COLUMN id SET DE ALTER TABLE ONLY analytics_cycle_analytics_project_value_streams ALTER COLUMN id SET DEFAULT nextval('analytics_cycle_analytics_project_value_streams_id_seq'::regclass); +ALTER TABLE ONLY analytics_cycle_analytics_stage_event_hashes ALTER COLUMN id SET DEFAULT nextval('analytics_cycle_analytics_stage_event_hashes_id_seq'::regclass); + ALTER TABLE ONLY analytics_devops_adoption_segments ALTER COLUMN id SET DEFAULT nextval('analytics_devops_adoption_segments_id_seq'::regclass); ALTER TABLE ONLY analytics_devops_adoption_snapshots ALTER COLUMN id SET DEFAULT nextval('analytics_devops_adoption_snapshots_id_seq'::regclass); @@ -21044,6 +21062,9 @@ ALTER TABLE ONLY analytics_cycle_analytics_project_stages ALTER TABLE ONLY analytics_cycle_analytics_project_value_streams ADD CONSTRAINT analytics_cycle_analytics_project_value_streams_pkey PRIMARY KEY (id); +ALTER TABLE ONLY analytics_cycle_analytics_stage_event_hashes + ADD CONSTRAINT analytics_cycle_analytics_stage_event_hashes_pkey PRIMARY KEY (id); + ALTER TABLE ONLY analytics_devops_adoption_segments ADD CONSTRAINT analytics_devops_adoption_segments_pkey PRIMARY KEY (id); @@ -23536,6 +23557,8 @@ CREATE INDEX index_custom_emoji_on_creator_id ON custom_emoji USING btree (creat CREATE UNIQUE INDEX index_custom_emoji_on_namespace_id_and_name ON custom_emoji USING btree (namespace_id, name); +CREATE UNIQUE INDEX index_cycle_analytics_stage_event_hashes_on_hash_sha_256 ON analytics_cycle_analytics_stage_event_hashes USING btree (hash_sha256); + CREATE UNIQUE INDEX index_daily_build_group_report_results_unique_columns ON ci_daily_build_group_report_results USING btree (project_id, ref_path, date, group_name); CREATE INDEX index_dast_profile_schedules_active_next_run_at ON dast_profile_schedules USING btree (active, next_run_at); @@ -23960,6 +23983,8 @@ CREATE INDEX index_group_repository_storage_moves_on_group_id ON group_repositor CREATE UNIQUE INDEX index_group_stages_on_group_id_group_value_stream_id_and_name ON analytics_cycle_analytics_group_stages USING btree (group_id, group_value_stream_id, name); +CREATE INDEX index_group_stages_on_stage_event_hash_id ON analytics_cycle_analytics_group_stages USING btree (stage_event_hash_id); + CREATE UNIQUE INDEX index_group_wiki_repositories_on_disk_path ON group_wiki_repositories USING btree (disk_path); CREATE INDEX index_group_wiki_repositories_on_shard_id ON group_wiki_repositories USING btree (shard_id); @@ -24762,6 +24787,8 @@ CREATE INDEX index_project_settings_on_project_id_partially ON project_settings CREATE UNIQUE INDEX index_project_settings_on_push_rule_id ON project_settings USING btree (push_rule_id); +CREATE INDEX index_project_stages_on_stage_event_hash_id ON analytics_cycle_analytics_project_stages USING btree (stage_event_hash_id); + CREATE INDEX index_project_statistics_on_namespace_id ON project_statistics USING btree (namespace_id); CREATE INDEX index_project_statistics_on_packages_size_and_project_id ON project_statistics USING btree (packages_size, project_id); @@ -26073,6 +26100,9 @@ ALTER TABLE ONLY members ALTER TABLE ONLY lfs_objects_projects ADD CONSTRAINT fk_2eb33f7a78 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE NOT VALID; +ALTER TABLE ONLY analytics_cycle_analytics_group_stages + ADD CONSTRAINT fk_3078345d6d FOREIGN KEY (stage_event_hash_id) REFERENCES analytics_cycle_analytics_stage_event_hashes(id) ON DELETE CASCADE; + ALTER TABLE ONLY lists ADD CONSTRAINT fk_30f2a831f4 FOREIGN KEY (iteration_id) REFERENCES sprints(id) ON DELETE CASCADE; @@ -26514,6 +26544,9 @@ ALTER TABLE ONLY packages_packages ALTER TABLE ONLY geo_event_log ADD CONSTRAINT fk_c1f241c70d FOREIGN KEY (upload_deleted_event_id) REFERENCES geo_upload_deleted_events(id) ON DELETE CASCADE; +ALTER TABLE ONLY analytics_cycle_analytics_project_stages + ADD CONSTRAINT fk_c3339bdfc9 FOREIGN KEY (stage_event_hash_id) REFERENCES analytics_cycle_analytics_stage_event_hashes(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_exports ADD CONSTRAINT fk_c3d3cb5d0f FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index d44d3f64eb1..48bd812c7f2 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -123,6 +123,9 @@ From there, you can see the following actions: - Created, updated, or deleted DAST profiles, DAST scanner profiles, and DAST site profiles ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1) - Changed a project's compliance framework ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329362) in GitLab 14.1) +- User password required for approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) +- Permission to modify merge requests approval rules in merge requests was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) +- New approvals requirement when new commits are added to an MR was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events). diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb index aa68833bdb8..adef4d1636a 100644 --- a/lib/gitlab/ci/config/entry/inherit/variables.rb +++ b/lib/gitlab/ci/config/entry/inherit/variables.rb @@ -13,9 +13,6 @@ module Gitlab strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) } class BooleanStrategy < ::Gitlab::Config::Entry::Boolean - def inherit?(_key) - value - end end class ArrayStrategy < ::Gitlab::Config::Entry::Node @@ -25,20 +22,12 @@ module Gitlab validates :config, type: Array validates :config, array_of_strings: true end - - def inherit?(key) - value.include?(key.to_s) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} should be a bool or array of strings"] end - - def inherit?(key) - false - end end end end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb index fba4b9e433a..408a901f69d 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -71,7 +71,7 @@ module Gitlab 'continuous_delivery' ], [ - %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), + %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), 'environments', 'continuous_delivery' ], diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 568e894a02f..25895dc6728 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -5,6 +5,8 @@ module Gitlab class Tag < Ref extend Gitlab::EncodingHelper + delegate :id, to: :@raw_tag + attr_reader :object_sha, :repository MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes @@ -24,6 +26,18 @@ module Gitlab def get_messages(repository, tag_ids) repository.gitaly_ref_client.get_tag_messages(tag_ids) end + + def extract_signature_lazily(repository, tag_id) + BatchLoader.for(tag_id).batch(key: repository) do |tag_ids, loader, args| + batch_signature_extraction(args[:key], tag_ids).each do |tag_id, signature_data| + loader.call(tag_id, signature_data) + end + end + end + + def batch_signature_extraction(repository, tag_ids) + repository.gitaly_ref_client.get_tag_signatures(tag_ids) + end end def initialize(repository, raw_tag) @@ -81,7 +95,7 @@ module Gitlab when :PGP nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260 when :X509 - X509::Tag.new(@raw_tag).signature + X509::Tag.new(@repository, self).signature else nil end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 90b5113abaf..7097d5bd181 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -178,6 +178,27 @@ module Gitlab messages end + def get_tag_signatures(tag_ids) + request = Gitaly::GetTagSignaturesRequest.new(repository: @gitaly_repo, tag_revisions: tag_ids) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) + + signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } + current_tag_id = nil + + response.each do |message| + message.signatures.each do |tag_signature| + current_tag_id = tag_signature.tag_id if tag_signature.tag_id.present? + + signatures[current_tag_id].first << tag_signature.signature + signatures[current_tag_id].last << tag_signature.content + end + end + + signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex + end + def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 7e45cd216f5..5e57642333f 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -14,7 +14,7 @@ module Gitlab Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout ].freeze HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ - SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep ].freeze diff --git a/lib/gitlab/signed_tag.rb b/lib/gitlab/signed_tag.rb new file mode 100644 index 00000000000..3b22cb7622d --- /dev/null +++ b/lib/gitlab/signed_tag.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + class SignedTag + include Gitlab::Utils::StrongMemoize + + def initialize(repository, tag) + @repository = repository + @tag = tag + + if Feature.enabled?(:get_tag_signatures) + @signature_data = Gitlab::Git::Tag.extract_signature_lazily(repository, tag.id) if repository + else + @signature_data = [signature_text_of_message.b, signed_text_of_message.b] + end + end + + def signature + return unless @tag.has_signature? + end + + def signature_text + @signature_data&.fetch(0) + end + + def signed_text + @signature_data&.fetch(1) + end + + private + + def signature_text_of_message + @tag.message.slice(@tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) + rescue StandardError + nil + end + + def signed_text_of_message + %{object #{@tag.target_commit.id} +type commit +tag #{@tag.name} +tagger #{@tag.tagger.name} <#{@tag.tagger.email}> #{@tag.tagger.date.seconds} #{@tag.tagger.timezone} + +#{@tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} + end + end +end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index ad85b200130..cf24e6f62bd 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -4,37 +4,16 @@ require 'digest' module Gitlab module X509 - class Tag + class Tag < Gitlab::SignedTag include Gitlab::Utils::StrongMemoize - def initialize(raw_tag) - @raw_tag = raw_tag - end - def signature - signature = X509::Signature.new(signature_text, signed_text, @raw_tag.tagger.email, Time.at(@raw_tag.tagger.date.seconds)) + strong_memoize(:signature) do + super - return if signature.verified_signature.nil? - - signature - end - - private - - def signature_text - @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) - rescue StandardError - nil - end - - def signed_text - # signed text is reconstructed as long as there is no specific gitaly function - %{object #{@raw_tag.target_commit.id} -type commit -tag #{@raw_tag.name} -tagger #{@raw_tag.tagger.name} <#{@raw_tag.tagger.email}> #{@raw_tag.tagger.date.seconds} #{@raw_tag.tagger.timezone} - -#{@raw_tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} + signature = X509::Signature.new(signature_text, signed_text, @tag.tagger.email, Time.at(@tag.tagger.date.seconds)) + signature unless signature.verified_signature.nil? + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0296f743e37..463dcbfab82 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2056,6 +2056,9 @@ msgstr "" msgid "Add system hook" msgstr "" +msgid "Add text to the sign-in page. Markdown enabled." +msgstr "" + msgid "Add to Slack" msgstr "" @@ -2872,9 +2875,6 @@ msgstr "" msgid "After a successful password update, you will be redirected to the login page where you can log in with your new password." msgstr "" -msgid "After sign-out path" -msgstr "" - msgid "After that, you will not be able to use merge approvals or code quality as well as many other features." msgstr "" @@ -3327,6 +3327,12 @@ msgstr "" msgid "Allow owners to manually add users outside of LDAP" msgstr "" +msgid "Allow password authentication for Git over HTTP(S)" +msgstr "" + +msgid "Allow password authentication for the web interface" +msgstr "" + msgid "Allow project maintainers to configure repository mirroring" msgstr "" @@ -11583,6 +11589,12 @@ msgstr "" msgid "Direct member" msgstr "" +msgid "Direct non-authenticated users to this page." +msgstr "" + +msgid "Direct users to this page after they sign out." +msgstr "" + msgid "Direction" msgstr "" @@ -12275,6 +12287,9 @@ msgstr "" msgid "Enable access to the performance bar for non-administrators in a given group." msgstr "" +msgid "Enable admin mode" +msgstr "" + msgid "Enable and disable Service Desk. Some additional configuration might be required. %{link_start}Learn more%{link_end}." msgstr "" @@ -12296,6 +12311,9 @@ msgstr "" msgid "Enable delayed project deletion by default for newly-created groups." msgstr "" +msgid "Enable email notification" +msgstr "" + msgid "Enable error tracking" msgstr "" @@ -12410,7 +12428,7 @@ msgstr "" msgid "Enabled Git access protocols" msgstr "" -msgid "Enabled OAuth sign-in sources" +msgid "Enabled OAuth authentication sources" msgstr "" msgid "Enabled sources for code import during project creation. OmniAuth must be configured for GitHub" @@ -12446,6 +12464,12 @@ msgstr "" msgid "Enforce personal access token expiration" msgstr "" +msgid "Enforce two-factor authentication" +msgstr "" + +msgid "Enforce two-factor authentication for all user sign-ins." +msgstr "" + msgid "Ensure connectivity is available from the GitLab server to the Prometheus server" msgstr "" @@ -20544,6 +20568,9 @@ msgstr "" msgid "Maximum time for web terminal websocket connection (in seconds). 0 for unlimited." msgstr "" +msgid "Maximum time that users are allowed to skip the setup of two-factor authentication (in hours). Set to 0 (zero) to enforce at next sign in." +msgstr "" + msgid "May" msgstr "" @@ -22864,7 +22891,7 @@ msgstr "" msgid "Notifications on" msgstr "" -msgid "Notify users by email when sign-in location is not recognized" +msgid "Notify users by email when sign-in location is not recognized." msgstr "" msgid "Nov" @@ -23952,12 +23979,6 @@ msgstr "" msgid "Password (optional)" msgstr "" -msgid "Password authentication enabled for Git over HTTP(S)" -msgstr "" - -msgid "Password authentication enabled for web interface" -msgstr "" - msgid "Password authentication is unavailable." msgstr "" @@ -28152,7 +28173,7 @@ msgstr "" msgid "Requests to these domain(s)/address(es) on the local network will be allowed when local requests from hooks and services are not allowed. IP ranges such as 1:0:0:0:0:0:0:0/124 or 127.0.0.0/28 are supported. Domain wildcards are not supported currently. Use comma, semicolon, or newline to separate multiple entries. The allowlist can hold a maximum of 1000 entries. Domains should use IDNA encoding. Ex: example.com, 192.168.1.1, 127.0.0.0/28, xn--itlab-j1a.com." msgstr "" -msgid "Require additional authentication for administrative tasks" +msgid "Require additional authentication for administrative tasks." msgstr "" msgid "Require all users in this group to setup Two-factor authentication" @@ -28161,9 +28182,6 @@ msgstr "" msgid "Require all users in this group to setup two-factor authentication" msgstr "" -msgid "Require all users to set up two-factor authentication" -msgstr "" - msgid "Required approvals (%{approvals_given} given)" msgstr "" @@ -30219,10 +30237,10 @@ msgstr "" msgid "Set projects and maximum size limits, session duration, user options, and check feature availability for namespace plan." msgstr "" -msgid "Set requirements for a user to sign-in. Enable mandatory two-factor authentication." +msgid "Set severity" msgstr "" -msgid "Set severity" +msgid "Set sign-in restrictions for all users." msgstr "" msgid "Set size limits for displaying diffs in the browser." @@ -30661,6 +30679,9 @@ msgstr "" msgid "Sign-in text" msgstr "" +msgid "Sign-out page URL" +msgstr "" + msgid "Sign-up restrictions" msgstr "" @@ -35191,7 +35212,7 @@ msgstr "" msgid "Two-factor authentication is not enabled for this user" msgstr "" -msgid "Two-factor grace period (hours)" +msgid "Two-factor grace period" msgstr "" msgid "Type" @@ -37016,12 +37037,6 @@ msgstr "" msgid "We will notify %{inviter} that you declined their invitation to join GitLab. You will stop receiving reminders." msgstr "" -msgid "We will redirect non-logged in users to this page" -msgstr "" - -msgid "We will redirect users to this page after they sign out" -msgstr "" - msgid "We would like to inform you that your subscription GitLab Enterprise Edition %{plan_name} is nearing its user limit. You have %{active_user_count} active users, which is almost at the user limit of %{maximum_user_count}." msgstr "" @@ -37298,7 +37313,7 @@ msgstr "" msgid "When an event in GitLab triggers a webhook, you can use the request details to figure out if something went wrong." msgstr "" -msgid "When disabled, an external authentication provider must be used." +msgid "When inactive, an external authentication provider must be used." msgstr "" msgid "When leaving the URL blank, classification labels can still be specified without disabling cross project features or performing external authorization checks." diff --git a/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb index b1a8fbcdbe0..bdb4d25c142 100644 --- a/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/inherit/variables_spec.rb @@ -24,19 +24,4 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Inherit::Variables do end end end - - describe '#inherit?' do - where(:config, :inherit) do - true | true - false | false - %w[A] | true - %w[B] | false - end - - with_them do - it do - expect(subject.inherit?('A')).to eq(inherit) - end - end - end end diff --git a/spec/lib/gitlab/etag_caching/router/restful_spec.rb b/spec/lib/gitlab/etag_caching/router/restful_spec.rb index 877789b320f..1f5cac09b6d 100644 --- a/spec/lib/gitlab/etag_caching/router/restful_spec.rb +++ b/spec/lib/gitlab/etag_caching/router/restful_spec.rb @@ -87,12 +87,18 @@ RSpec.describe Gitlab::EtagCaching::Router::Restful do end it 'matches the environments path' do - result = match_route('/my-group/my-project/environments.json') + result = match_route('/my-group/my-project/-/environments.json') expect(result).to be_present expect(result.name).to eq 'environments' end + it 'does not match the operations environments list path' do + result = match_route('/-/operations/environments.json') + + expect(result).not_to be_present + end + it 'matches pipeline#show endpoint' do result = match_route('/my-group/my-project/-/pipelines/2.json') diff --git a/spec/lib/gitlab/git/tag_spec.rb b/spec/lib/gitlab/git/tag_spec.rb index b6ff76c5e1c..79ae47f8a7b 100644 --- a/spec/lib/gitlab/git/tag_spec.rb +++ b/spec/lib/gitlab/git/tag_spec.rb @@ -38,7 +38,7 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do it { expect(tag.tagger.timezone).to eq("+0200") } end - describe 'signed tag' do + shared_examples 'signed tag' do let(:project) { create(:project, :repository) } let(:tag) { project.repository.find_tag('v1.1.1') } @@ -54,6 +54,18 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do it { expect(tag.tagger.timezone).to eq("+0100") } end + context 'with :get_tag_signatures enabled' do + it_behaves_like 'signed tag' + end + + context 'with :get_tag_signatures disabled' do + before do + stub_feature_flags(get_tag_signatures: false) + end + + it_behaves_like 'signed tag' + end + it { expect(repository.tags.size).to eq(SeedRepo::Repo::TAGS.size) } end @@ -77,6 +89,75 @@ RSpec.describe Gitlab::Git::Tag, :seed_helper do end end + describe '.extract_signature_lazily' do + let(:project) { create(:project, :repository) } + + subject { described_class.extract_signature_lazily(project.repository, tag_id).itself } + + context 'when the tag is signed' do + let(:tag_id) { project.repository.find_tag('v1.1.1').id } + + it 'returns signature and signed text' do + signature, signed_text = subject + + expect(signature).to eq(X509Helpers::User1.signed_tag_signature.chomp) + expect(signature).to be_a_binary_string + expect(signed_text).to eq(X509Helpers::User1.signed_tag_base_data) + expect(signed_text).to be_a_binary_string + end + end + + context 'when the tag has no signature' do + let(:tag_id) { project.repository.find_tag('v1.0.0').id } + + it 'returns empty signature and message as signed text' do + signature, signed_text = subject + + expect(signature).to be_empty + expect(signed_text).to eq(X509Helpers::User1.unsigned_tag_base_data) + expect(signed_text).to be_a_binary_string + end + end + + context 'when the tag cannot be found' do + let(:tag_id) { Gitlab::Git::BLANK_SHA } + + it 'raises GRPC::Internal' do + expect { subject }.to raise_error(GRPC::Internal) + end + end + + context 'when the tag ID is invalid' do + let(:tag_id) { '4b4918a572fa86f9771e5ba40fbd48e' } + + it 'raises GRPC::Internal' do + expect { subject }.to raise_error(GRPC::Internal) + end + end + + context 'when loading signatures in batch once' do + it 'fetches signatures in batch once' do + tag_ids = [project.repository.find_tag('v1.1.1').id, project.repository.find_tag('v1.0.0').id] + signatures = tag_ids.map do |tag_id| + described_class.extract_signature_lazily(repository, tag_id) + end + + other_repository = double(:repository) + described_class.extract_signature_lazily(other_repository, tag_ids.first) + + expect(described_class).to receive(:batch_signature_extraction) + .with(repository, tag_ids) + .once + .and_return({}) + + expect(described_class).not_to receive(:batch_signature_extraction) + .with(other_repository, tag_ids.first) + + 2.times { signatures.each(&:itself) } + end + end + end + describe 'tag into from Gitaly tag' do context 'message_size != message.size' do let(:gitaly_tag) { build(:gitaly_tag, message: ''.b, message_size: message_size) } diff --git a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb index a4c6e30bba8..e19be965e68 100644 --- a/spec/lib/gitlab/gitaly_client/ref_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_service_spec.rb @@ -178,6 +178,17 @@ RSpec.describe Gitlab::GitalyClient::RefService do end end + describe '#get_tag_signatures' do + it 'sends a get_tag_signatures message' do + expect_any_instance_of(Gitaly::RefService::Stub) + .to receive(:get_tag_signatures) + .with(gitaly_request_with_params(tag_revisions: ['some_tag_id']), kind_of(Hash)) + .and_return([]) + + client.get_tag_signatures(['some_tag_id']) + end + end + describe '#find_ref_name', :seed_helper do subject { client.find_ref_name(SeedRepo::Commit::ID, 'refs/heads/master') } diff --git a/spec/lib/gitlab/usage_data_metrics_spec.rb b/spec/lib/gitlab/usage_data_metrics_spec.rb index 5bc72845222..7b6b8dacc4c 100644 --- a/spec/lib/gitlab/usage_data_metrics_spec.rb +++ b/spec/lib/gitlab/usage_data_metrics_spec.rb @@ -87,6 +87,24 @@ RSpec.describe Gitlab::UsageDataMetrics do ]) end + it 'includes terraform monthly key' do + expect(subject[:redis_hll_counters][:terraform].keys).to include(:p_terraform_state_api_unique_users_monthly) + end + + it 'includes terraform monthly and weekly keys' do + expect(subject[:redis_hll_counters][:pipeline_authoring].keys).to contain_exactly(*[ + :o_pipeline_authoring_unique_users_committing_ciconfigfile_monthly, :o_pipeline_authoring_unique_users_committing_ciconfigfile_weekly, + :o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_monthly, :o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile_weekly, + :pipeline_authoring_total_unique_counts_monthly, :pipeline_authoring_total_unique_counts_weekly + ]) + end + + it 'includes users_expanding_secure_security_report monthly and weekly keys' do + expect(subject[:redis_hll_counters][:secure].keys).to contain_exactly(*[ + :users_expanding_secure_security_report_monthly, :users_expanding_secure_security_report_weekly + ]) + end + it 'includes issues_edit monthly and weekly keys' do expect(subject[:redis_hll_counters][:issues_edit].keys).to include( :g_project_management_issue_title_changed_monthly, :g_project_management_issue_title_changed_weekly, diff --git a/spec/lib/gitlab/x509/tag_spec.rb b/spec/lib/gitlab/x509/tag_spec.rb index b011ea515de..be120aaf16a 100644 --- a/spec/lib/gitlab/x509/tag_spec.rb +++ b/spec/lib/gitlab/x509/tag_spec.rb @@ -2,13 +2,13 @@ require 'spec_helper' RSpec.describe Gitlab::X509::Tag do - subject(:signature) { described_class.new(tag).signature } + subject(:signature) { described_class.new(project.repository, tag).signature } describe '#signature' do let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '', 'group/project') } let(:project) { create(:project, :repository) } - describe 'signed tag' do + shared_examples 'signed tag' do let(:tag) { project.repository.find_tag('v1.1.1') } let(:certificate_attributes) do { @@ -33,10 +33,24 @@ RSpec.describe Gitlab::X509::Tag do it { expect(signature.x509_certificate.x509_issuer).to have_attributes(issuer_attributes) } end - context 'unsigned tag' do + shared_examples 'unsigned tag' do let(:tag) { project.repository.find_tag('v1.0.0') } it { expect(signature).to be_nil } end + + context 'with :get_tag_signatures enabled' do + it_behaves_like 'signed tag' + it_behaves_like 'unsigned tag' + end + + context 'with :get_tag_signatures disabled' do + before do + stub_feature_flags(get_tag_signatures: false) + end + + it_behaves_like 'signed tag' + it_behaves_like 'unsigned tag' + end end end diff --git a/spec/models/analytics/cycle_analytics/stage_event_hash_spec.rb b/spec/models/analytics/cycle_analytics/stage_event_hash_spec.rb new file mode 100644 index 00000000000..ffddaf1e1b2 --- /dev/null +++ b/spec/models/analytics/cycle_analytics/stage_event_hash_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Analytics::CycleAnalytics::StageEventHash, type: :model do + let(:stage_event_hash) { described_class.create!(hash_sha256: hash_sha256) } + let(:hash_sha256) { 'does_not_matter' } + + describe 'associations' do + it { is_expected.to have_many(:cycle_analytics_project_stages) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:hash_sha256) } + end + + describe '.record_id_by_hash_sha256' do + it 'returns an existing id' do + id = stage_event_hash.id + same_id = described_class.record_id_by_hash_sha256(hash_sha256) + + expect(same_id).to eq(id) + end + + it 'creates a new record' do + expect do + described_class.record_id_by_hash_sha256(hash_sha256) + end.to change { described_class.count }.from(0).to(1) + end + end + + describe '.cleanup_if_unused' do + it 'removes the record' do + described_class.cleanup_if_unused(stage_event_hash.id) + + expect(described_class.find_by_id(stage_event_hash.id)).to be_nil + end + + it 'does not remove the record' do + id = create(:cycle_analytics_project_stage).stage_event_hash_id + + described_class.cleanup_if_unused(id) + + expect(described_class.find_by_id(id)).not_to be_nil + end + end +end diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index a6beb12886b..f36b0a62aa3 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -840,6 +840,8 @@ RSpec.describe ProjectPolicy do it { is_expected.to be_allowed(:read_package) } it { is_expected.to be_allowed(:read_project) } it { is_expected.to be_disallowed(:create_package) } + + it_behaves_like 'package access with repository disabled' end context 'a deploy token with write_package_registry scope' do @@ -849,6 +851,8 @@ RSpec.describe ProjectPolicy do it { is_expected.to be_allowed(:read_package) } it { is_expected.to be_allowed(:read_project) } it { is_expected.to be_disallowed(:destroy_package) } + + it_behaves_like 'package access with repository disabled' end end @@ -1021,18 +1025,7 @@ RSpec.describe ProjectPolicy do it { is_expected.to be_allowed(:read_package) } - context 'when repository is disabled' do - before do - project.project_feature.update!( - # Disable merge_requests and builds as well, since merge_requests and - # builds cannot have higher visibility than repository. - merge_requests_access_level: ProjectFeature::DISABLED, - builds_access_level: ProjectFeature::DISABLED, - repository_access_level: ProjectFeature::DISABLED) - end - - it { is_expected.to be_disallowed(:read_package) } - end + it_behaves_like 'package access with repository disabled' end context 'with owner' do diff --git a/spec/requests/api/maven_packages_spec.rb b/spec/requests/api/maven_packages_spec.rb index 2bb6d05f54b..c3fd02dad51 100644 --- a/spec/requests/api/maven_packages_spec.rb +++ b/spec/requests/api/maven_packages_spec.rb @@ -217,6 +217,15 @@ RSpec.describe API::MavenPackages do end end + shared_examples 'successfully returning the file' do + it 'returns the file', :aggregate_failures do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq('application/octet-stream') + end + end + describe 'GET /api/v4/packages/maven/*path/:file_name' do context 'a public project' do subject { download_file(file_name: package_file.file_name) } @@ -224,12 +233,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'returns sha1 of the file' do download_file(file_name: package_file.file_name + '.sha1') @@ -260,12 +264,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'denies download when no private token' do download_file(file_name: package_file.file_name) @@ -297,12 +296,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'denies download when not enough permissions' do unless project.root_namespace == user.namespace @@ -409,12 +403,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file for a group' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'returns sha1 of the file' do download_file(file_name: package_file.file_name + '.sha1') @@ -445,12 +434,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file for a group' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'denies download when no private token' do download_file(file_name: package_file.file_name) @@ -482,12 +466,7 @@ RSpec.describe API::MavenPackages do shared_examples 'getting a file for a group' do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'denies download when not enough permissions' do group.add_guest(user) @@ -516,12 +495,7 @@ RSpec.describe API::MavenPackages do context 'with group deploy token' do subject { download_file_with_token(file_name: package_file.file_name, request_headers: group_deploy_token_headers) } - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'returns the file with only write_package_registry scope' do deploy_token_for_group.update!(read_package_registry: false) @@ -553,12 +527,7 @@ RSpec.describe API::MavenPackages do group.add_reporter(user) end - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' context 'with a non existing maven path' do subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3', request_headers: headers_with_token, group_id: root_group.id) } @@ -657,12 +626,7 @@ RSpec.describe API::MavenPackages do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'returns sha1 of the file' do download_file(file_name: package_file.file_name + '.sha1') @@ -672,6 +636,19 @@ RSpec.describe API::MavenPackages do expect(response.body).to eq(package_file.file_sha1) end + context 'when the repository is disabled' do + before do + project.project_feature.update!( + # Disable merge_requests and builds as well, since merge_requests and + # builds cannot have higher visibility than repository. + merge_requests_access_level: ProjectFeature::DISABLED, + builds_access_level: ProjectFeature::DISABLED, + repository_access_level: ProjectFeature::DISABLED) + end + + it_behaves_like 'successfully returning the file' + end + context 'with a non existing maven path' do subject { download_file(file_name: package_file.file_name, path: 'foo/bar/1.2.3') } @@ -688,12 +665,7 @@ RSpec.describe API::MavenPackages do it_behaves_like 'tracking the file download event' - it 'returns the file' do - subject - - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') - end + it_behaves_like 'successfully returning the file' it 'denies download when not enough permissions' do project.add_guest(user) diff --git a/spec/support/helpers/x509_helpers.rb b/spec/support/helpers/x509_helpers.rb index ce0fa268ace..1dc8b1d4845 100644 --- a/spec/support/helpers/x509_helpers.rb +++ b/spec/support/helpers/x509_helpers.rb @@ -290,6 +290,17 @@ module X509Helpers SIGNEDDATA end + def unsigned_tag_base_data + <<~SIGNEDDATA + object 6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9 + type commit + tag v1.0.0 + tagger Dmitriy Zaporozhets 1393491299 +0200 + + Release + SIGNEDDATA + end + def certificate_crl 'http://ch.siemens.com/pki?ZZZZZZA2.crl' end diff --git a/spec/support/shared_examples/models/cycle_analytics_stage_shared_examples.rb b/spec/support/shared_examples/models/cycle_analytics_stage_shared_examples.rb index cf38a583944..78aa4b0b809 100644 --- a/spec/support/shared_examples/models/cycle_analytics_stage_shared_examples.rb +++ b/spec/support/shared_examples/models/cycle_analytics_stage_shared_examples.rb @@ -13,6 +13,7 @@ RSpec.shared_examples 'value stream analytics stage' do describe 'associations' do it { is_expected.to belong_to(:end_event_label) } it { is_expected.to belong_to(:start_event_label) } + it { is_expected.to belong_to(:stage_event_hash) } end describe 'validation' do @@ -138,6 +139,67 @@ RSpec.shared_examples 'value stream analytics stage' do expect(stage_1.events_hash_code).not_to eq(stage_2.events_hash_code) end end + + # rubocop: disable Rails/SaveBang + describe '#event_hash' do + it 'associates the same stage event hash record' do + first = create(factory) + second = create(factory) + + expect(first.stage_event_hash_id).to eq(second.stage_event_hash_id) + end + + it 'does not introduce duplicated stage event hash records' do + expect do + create(factory) + create(factory) + end.to change { Analytics::CycleAnalytics::StageEventHash.count }.from(0).to(1) + end + + it 'creates different hash record for different event configurations' do + expect do + create(factory, start_event_identifier: :issue_created, end_event_identifier: :issue_first_mentioned_in_commit) + create(factory, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged) + end.to change { Analytics::CycleAnalytics::StageEventHash.count }.from(0).to(2) + end + + context 'when the stage event hash changes' do + let(:stage) { create(factory, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged) } + + it 'deletes the old, unused stage event hash record' do + old_stage_event_hash = stage.stage_event_hash + + stage.update!(end_event_identifier: :merge_request_first_deployed_to_production) + + expect(stage.stage_event_hash_id).not_to eq(old_stage_event_hash.id) + + old_stage_event_hash_from_db = Analytics::CycleAnalytics::StageEventHash.find_by_id(old_stage_event_hash.id) + expect(old_stage_event_hash_from_db).to be_nil + end + + it 'does not delete used stage event hash record' do + other_stage = create(factory, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged) + + stage.update!(end_event_identifier: :merge_request_first_deployed_to_production) + + expect(stage.stage_event_hash_id).not_to eq(other_stage.stage_event_hash_id) + + old_stage_event_hash_from_db = Analytics::CycleAnalytics::StageEventHash.find_by_id(other_stage.stage_event_hash_id) + expect(old_stage_event_hash_from_db).not_to be_nil + end + end + + context 'when the stage events hash code does not change' do + it 'does not trigger extra query on save' do + stage = create(factory, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged) + + expect(Analytics::CycleAnalytics::StageEventHash).not_to receive(:record_id_by_hash_sha256) + + stage.update!(name: 'new title') + end + end + end + # rubocop: enable Rails/SaveBang end RSpec.shared_examples 'value stream analytics label based stage' do diff --git a/spec/support/shared_examples/policies/project_policy_shared_examples.rb b/spec/support/shared_examples/policies/project_policy_shared_examples.rb index 013c9b61b99..a4243db6bc9 100644 --- a/spec/support/shared_examples/policies/project_policy_shared_examples.rb +++ b/spec/support/shared_examples/policies/project_policy_shared_examples.rb @@ -330,3 +330,18 @@ RSpec.shared_examples 'project policies as admin without admin mode' do end end end + +RSpec.shared_examples 'package access with repository disabled' do + context 'when repository is disabled' do + before do + project.project_feature.update!( + # Disable merge_requests and builds as well, since merge_requests and + # builds cannot have higher visibility than repository. + merge_requests_access_level: ProjectFeature::DISABLED, + builds_access_level: ProjectFeature::DISABLED, + repository_access_level: ProjectFeature::DISABLED) + end + + it { is_expected.to be_allowed(:read_package) } + end +end diff --git a/spec/workers/web_hook_worker_spec.rb b/spec/workers/web_hook_worker_spec.rb index a86964aa417..0f40177eb7d 100644 --- a/spec/workers/web_hook_worker_spec.rb +++ b/spec/workers/web_hook_worker_spec.rb @@ -15,6 +15,10 @@ RSpec.describe WebHookWorker do subject.perform(project_hook.id, data, hook_name) end + it 'does not error when the WebHook record cannot be found' do + expect { subject.perform(non_existing_record_id, data, hook_name) }.not_to raise_error + end + it_behaves_like 'worker with data consistency', described_class, data_consistency: :delayed diff --git a/tooling/bin/find_tests b/tooling/bin/find_tests index 2c0e7ae2c53..97fadf406fe 100755 --- a/tooling/bin/find_tests +++ b/tooling/bin/find_tests @@ -5,9 +5,10 @@ require 'gitlab' require 'test_file_finder' gitlab_token = ENV.fetch('DANGER_GITLAB_API_TOKEN', '') +gitlab_endpoint = ENV.fetch('CI_API_V4_URL') Gitlab.configure do |config| - config.endpoint = 'https://gitlab.com/api/v4' + config.endpoint = gitlab_endpoint config.private_token = gitlab_token end