From bc935f05bc8d7dd89c3e7c88f90264e90b636e07 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 7 Oct 2021 09:12:01 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../grouped_test_reports_app.vue | 6 +- .../projects/merge_requests_controller.rb | 1 - app/models/pages_domain.rb | 13 +-- ...ge_data_i_testing_summary_widget_total.yml | 8 -- .../database/efficient_in_operator_queries.md | 33 ++++++- .../email/hook/smime_signature_interceptor.rb | 2 +- lib/gitlab/email/smime/certificate.rb | 58 ----------- .../in_operator_optimization/query_builder.rb | 30 ++---- .../order_values_loader_strategy.rb | 38 +++++++ .../strategies/record_loader_strategy.rb | 42 ++++++++ .../known_events/common.yml | 1 - lib/gitlab/x509/certificate.rb | 56 +++++++++++ spec/factories/pages_domains.rb | 12 +++ spec/factories_spec.rb | 1 + spec/fixtures/ssl/letsencrypt_expired_x3.pem | 98 +++++++++++++++++++ .../grouped_test_reports_app_spec.js | 18 +--- .../hook/smime_signature_interceptor_spec.rb | 8 +- .../query_builder_spec.rb | 28 +++++- .../order_values_loader_strategy_spec.rb | 34 +++++++ .../strategies/record_loader_strategy_spec.rb | 60 ++++++++++++ .../{email/smime => x509}/certificate_spec.rb | 2 +- spec/models/pages_domain_spec.rb | 13 +++ 22 files changed, 435 insertions(+), 127 deletions(-) delete mode 100644 config/feature_flags/development/usage_data_i_testing_summary_widget_total.yml delete mode 100644 lib/gitlab/email/smime/certificate.rb create mode 100644 lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb create mode 100644 lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb create mode 100644 lib/gitlab/x509/certificate.rb create mode 100644 spec/fixtures/ssl/letsencrypt_expired_x3.pem create mode 100644 spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb create mode 100644 spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb rename spec/lib/gitlab/{email/smime => x509}/certificate_spec.rb (98%) diff --git a/app/assets/javascripts/reports/grouped_test_report/grouped_test_reports_app.vue b/app/assets/javascripts/reports/grouped_test_report/grouped_test_reports_app.vue index 82806793401..be49a03a9a5 100644 --- a/app/assets/javascripts/reports/grouped_test_report/grouped_test_reports_app.vue +++ b/app/assets/javascripts/reports/grouped_test_report/grouped_test_reports_app.vue @@ -3,7 +3,6 @@ import { GlButton, GlIcon } from '@gitlab/ui'; import { mapActions, mapGetters, mapState } from 'vuex'; import api from '~/api'; import { sprintf, s__ } from '~/locale'; -import glFeatureFlagsMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; import GroupedIssuesList from '../components/grouped_issues_list.vue'; import { componentNames } from '../components/issue_body'; import ReportSection from '../components/report_section.vue'; @@ -28,7 +27,6 @@ export default { GlButton, GlIcon, }, - mixins: [glFeatureFlagsMixin()], props: { endpoint: { type: String, @@ -82,9 +80,7 @@ export default { methods: { ...mapActions(['setPaths', 'fetchReports', 'closeModal']), handleToggleEvent() { - if (this.glFeatures.usageDataITestingSummaryWidgetTotal) { - api.trackRedisHllUserEvent(this.$options.expandEvent); - } + api.trackRedisHllUserEvent(this.$options.expandEvent); }, reportText(report) { const { name, summary } = report || {}; diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index 9a8eff56927..46df514abcb 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -37,7 +37,6 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo push_frontend_feature_flag(:core_security_mr_widget_counts, @project) push_frontend_feature_flag(:paginated_notes, @project, default_enabled: :yaml) push_frontend_feature_flag(:confidential_notes, @project, default_enabled: :yaml) - push_frontend_feature_flag(:usage_data_i_testing_summary_widget_total, @project, default_enabled: :yaml) push_frontend_feature_flag(:improved_emoji_picker, project, default_enabled: :yaml) push_frontend_feature_flag(:diffs_virtual_scrolling, project, default_enabled: :yaml) push_frontend_feature_flag(:restructured_mr_widget, project, default_enabled: :yaml) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index c932d0bf800..0c5a155d48a 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -129,18 +129,15 @@ class PagesDomain < ApplicationRecord store = OpenSSL::X509::Store.new store.set_default_paths - # This forces to load all intermediate certificates stored in `certificate` - Tempfile.open('certificate_chain') do |f| - f.write(certificate) - f.flush - store.add_file(f.path) - end - - store.verify(x509) + store.verify(x509, untrusted_ca_certs_bundle) rescue OpenSSL::X509::StoreError false end + def untrusted_ca_certs_bundle + ::Gitlab::X509::Certificate.load_ca_certs_bundle(certificate) + end + def expired? return false unless x509 diff --git a/config/feature_flags/development/usage_data_i_testing_summary_widget_total.yml b/config/feature_flags/development/usage_data_i_testing_summary_widget_total.yml deleted file mode 100644 index fb06ea9f58d..00000000000 --- a/config/feature_flags/development/usage_data_i_testing_summary_widget_total.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: usage_data_i_testing_summary_widget_total -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57543 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326058 -milestone: '13.11' -type: development -group: group::testing -default_enabled: true diff --git a/doc/development/database/efficient_in_operator_queries.md b/doc/development/database/efficient_in_operator_queries.md index bc72bce30bf..0e979534acd 100644 --- a/doc/development/database/efficient_in_operator_queries.md +++ b/doc/development/database/efficient_in_operator_queries.md @@ -226,7 +226,12 @@ Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new( - `finder_query` loads the actual record row from the database. It must also be a lambda, where the order by column expressions is available for locating the record. In this example, the yielded values are `created_at` and `id` SQL expressions. Finding a record is very fast via the - primary key, so we don't use the `created_at` value. + primary key, so we don't use the `created_at` value. Providing the `finder_query` lambda is optional. + If it's not given, the IN operator optimization will only make the ORDER BY columns available to + the end-user and not the full database row. + + If it's not given, the IN operator optimization will only make the ORDER BY columns available to + the end-user and not the full database row. The following database index on the `issues` table must be present to make the query execute efficiently: @@ -611,6 +616,32 @@ Gitlab::Pagination::Keyset::Iterator.new(scope: scope, **opts).each_batch(of: 10 end ``` +NOTE: +The query loads complete database rows from the disk. This may cause increased I/O and slower +database queries. Depending on the use case, the primary key is often only +needed for the batch query to invoke additional statements. For example, `UPDATE` or `DELETE`. The +`id` column is included in the `ORDER BY` columns (`created_at` and `id`) and is already +loaded. In this case, you can omit the `finder_query` parameter. + +Example for loading the `ORDER BY` columns only: + +```ruby +scope = Issue.order(:created_at, :id) +array_scope = Group.find(9970).all_projects.select(:id) +array_mapping_scope = -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) } + +opts = { + in_operator_optimization_options: { + array_scope: array_scope, + array_mapping_scope: array_mapping_scope + } +} + +Gitlab::Pagination::Keyset::Iterator.new(scope: scope, **opts).each_batch(of: 100) do |records| + puts records.select(:id).map { |r| [r.id] } # only id and created_at are available +end +``` + #### Keyset pagination The optimization works out of the box with GraphQL and the `keyset_paginate` helper method. diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb index fe39589d019..0b092b3e41e 100644 --- a/lib/gitlab/email/hook/smime_signature_interceptor.rb +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -22,7 +22,7 @@ module Gitlab private def certificate - @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path, ca_certs_path) + @certificate ||= Gitlab::X509::Certificate.from_files(key_path, cert_path, ca_certs_path) end def key_path diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb deleted file mode 100644 index 3607b95b4bc..00000000000 --- a/lib/gitlab/email/smime/certificate.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Smime - class Certificate - CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze - - attr_reader :key, :cert, :ca_certs - - def key_string - key.to_s - end - - def cert_string - cert.to_pem - end - - def ca_certs_string - ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank? - end - - def self.from_strings(key_string, cert_string, ca_certs_string = nil) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - ca_certs = load_ca_certs_bundle(ca_certs_string) - - new(key, cert, ca_certs) - end - - def self.from_files(key_path, cert_path, ca_certs_path = nil) - ca_certs_string = File.read(ca_certs_path) if ca_certs_path - - from_strings(File.read(key_path), File.read(cert_path), ca_certs_string) - end - - # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found - # - # Ruby OpenSSL::X509::Certificate.new will only load the first - # certificate if a bundle is presented, this allows to parse multiple certs - # in the same file - def self.load_ca_certs_bundle(ca_certs_string) - return [] unless ca_certs_string - - ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| - OpenSSL::X509::Certificate.new(ca_cert_string) - end - end - - def initialize(key, cert, ca_certs = nil) - @key = key - @cert = cert - @ca_certs = ca_certs - end - end - end - end -end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 39d6e016ac7..c4cf6dfbf88 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -9,7 +9,6 @@ module Gitlab UnsupportedScopeOrder = Class.new(StandardError) RECURSIVE_CTE_NAME = 'recursive_keyset_cte' - RECORDS_COLUMN = 'records' # This class optimizes slow database queries (PostgreSQL specific) where the # IN SQL operator is used with sorting. @@ -42,7 +41,7 @@ module Gitlab # > array_mapping_scope: array_mapping_scope, # > finder_query: finder_query # > ).execute.limit(20) - def initialize(scope:, array_scope:, array_mapping_scope:, finder_query:, values: {}) + def initialize(scope:, array_scope:, array_mapping_scope:, finder_query: nil, values: {}) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) unless success @@ -57,11 +56,11 @@ module Gitlab @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @array_scope = array_scope @array_mapping_scope = array_mapping_scope - @finder_query = finder_query @values = values @model = @scope.model @table_name = @model.table_name @arel_table = @model.arel_table + @finder_strategy = finder_query.present? ? Strategies::RecordLoaderStrategy.new(finder_query, model, order_by_columns) : Strategies::OrderValuesLoaderStrategy.new(model, order_by_columns) end def execute @@ -74,7 +73,7 @@ module Gitlab q = cte .apply_to(model.where({}) .with(selector_cte.to_arel)) - .select(result_collector_final_projections) + .select(finder_strategy.final_projections) .where("count <> 0") # filter out the initializer row model.from(q.arel.as(table_name)) @@ -82,13 +81,13 @@ module Gitlab private - attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_query, :values, :model, :table_name, :arel_table + attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_strategy, :values, :model, :table_name, :arel_table def initializer_query array_column_names = array_scope_columns.array_aggregated_column_names + order_by_columns.array_aggregated_column_names projections = [ - *result_collector_initializer_columns, + *finder_strategy.initializer_columns, *array_column_names, '0::bigint AS count' ] @@ -156,7 +155,7 @@ module Gitlab order_column_value_arrays = order_by_columns.replace_value_in_array_by_position_expressions select = [ - *result_collector_columns, + *finder_strategy.columns, *array_column_list, *order_column_value_arrays, "#{RECURSIVE_CTE_NAME}.count + 1" @@ -254,23 +253,6 @@ module Gitlab end.join(", ") end - def result_collector_initializer_columns - ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] - end - - def result_collector_columns - query = finder_query - .call(*order_by_columns.array_lookup_expressions_by_position(RECURSIVE_CTE_NAME)) - .select("#{table_name}") - .limit(1) - - ["(#{query.to_sql})"] - end - - def result_collector_final_projections - ["(#{RECORDS_COLUMN}).*"] - end - def array_scope_columns @array_scope_columns ||= ArrayScopeColumns.new(array_scope.select_values) end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb new file mode 100644 index 00000000000..fc2b56048f6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class OrderValuesLoaderStrategy + def initialize(model, order_by_columns) + @model = model + @order_by_columns = order_by_columns + end + + def initializer_columns + order_by_columns.map do |column| + column_name = column.original_column_name.to_s + type = model.columns_hash[column_name].sql_type + "NULL::#{type} AS #{column_name}" + end + end + + def columns + order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME) + end + + def final_projections + order_by_columns.map(&:original_column_name) + end + + private + + attr_reader :model, :order_by_columns + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb new file mode 100644 index 00000000000..b12c33d6e51 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class RecordLoaderStrategy + RECORDS_COLUMN = 'records' + + def initialize(finder_query, model, order_by_columns) + @finder_query = finder_query + @order_by_columns = order_by_columns + @table_name = model.table_name + end + + def initializer_columns + ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] + end + + def columns + query = finder_query + .call(*order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME)) + .select("#{table_name}") + .limit(1) + + ["(#{query.to_sql})"] + end + + def final_projections + ["(#{RECORDS_COLUMN}).*"] + end + + private + + attr_reader :finder_query, :order_by_columns, :table_name + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index dc1ce8fd08a..feebc7f395a 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -181,7 +181,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_summary_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb new file mode 100644 index 00000000000..c7289a51b49 --- /dev/null +++ b/lib/gitlab/x509/certificate.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module X509 + class Certificate + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + + attr_reader :key, :cert, :ca_certs + + def key_string + key.to_s + end + + def cert_string + cert.to_pem + end + + def ca_certs_string + ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank? + end + + def self.from_strings(key_string, cert_string, ca_certs_string = nil) + key = OpenSSL::PKey::RSA.new(key_string) + cert = OpenSSL::X509::Certificate.new(cert_string) + ca_certs = load_ca_certs_bundle(ca_certs_string) + + new(key, cert, ca_certs) + end + + def self.from_files(key_path, cert_path, ca_certs_path = nil) + ca_certs_string = File.read(ca_certs_path) if ca_certs_path + + from_strings(File.read(key_path), File.read(cert_path), ca_certs_string) + end + + # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found + # + # Ruby OpenSSL::X509::Certificate.new will only load the first + # certificate if a bundle is presented, this allows to parse multiple certs + # in the same file + def self.load_ca_certs_bundle(ca_certs_string) + return [] unless ca_certs_string + + ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| + OpenSSL::X509::Certificate.new(ca_cert_string) + end + end + + def initialize(key, cert, ca_certs = nil) + @key = key + @cert = cert + @ca_certs = ca_certs + end + end + end +end diff --git a/spec/factories/pages_domains.rb b/spec/factories/pages_domains.rb index 2ba5cbb48bf..f3f2af79b76 100644 --- a/spec/factories/pages_domains.rb +++ b/spec/factories/pages_domains.rb @@ -258,6 +258,18 @@ ZDXgrA== certificate_source { :gitlab_provided } end + # This contains: + # webdioxide.com + # Let's Encrypt R3 + # ISRG Root X1 (issued by DST Root CA X3) + # + # DST Root CA X3 expired on 2021-09-30, but ISRG Root X1 should be trusted on most systems. + trait :letsencrypt_expired_x3_root do + certificate do + File.read(Rails.root.join('spec/fixtures/ssl', 'letsencrypt_expired_x3.pem')) + end + end + trait :explicit_ecdsa do certificate do '-----BEGIN CERTIFICATE----- diff --git a/spec/factories_spec.rb b/spec/factories_spec.rb index 6c7c3776c4a..7dc38b25fac 100644 --- a/spec/factories_spec.rb +++ b/spec/factories_spec.rb @@ -29,6 +29,7 @@ RSpec.describe 'factories' do [:pages_domain, :with_trusted_chain], [:pages_domain, :with_trusted_expired_chain], [:pages_domain, :explicit_ecdsa], + [:pages_domain, :letsencrypt_expired_x3_root], [:project_member, :blocked], [:remote_mirror, :ssh], [:user_preference, :only_comments], diff --git a/spec/fixtures/ssl/letsencrypt_expired_x3.pem b/spec/fixtures/ssl/letsencrypt_expired_x3.pem new file mode 100644 index 00000000000..462df721ed7 --- /dev/null +++ b/spec/fixtures/ssl/letsencrypt_expired_x3.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIGJDCCBQygAwIBAgISBOSAE/WwQGsTbDJI1vDL9+eKMA0GCSqGSIb3DQEBCwUA +MDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQD +EwJSMzAeFw0yMTEwMDEyMjIxMTlaFw0yMTEyMzAyMjIxMThaMBkxFzAVBgNVBAMT +DndlYmRpb3hpZGUuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +wf/TpE5AjzoLXMFQ+WHle7Dn5rlEe0bPee2JU386cZmMYnGFS5DR251FerSX28U4 +pqk2yS8oefHGi2PS6h8/MWxr+Zy/6hk3WkgwdIK3uPiUcfCdPV/btXDd4YqikEDm +BoOE4fQlqKQwtLOnhEZu9y8FQoxxoQ+7DndHrDixDoMbpUloxpqUZwziQnH4QHXE +32rQhq25+NUK/lVFGKOFnmZ2s/yUildKafqulHrLHOhumKMOEivzlFDZbtqP+RKt +nsrJ3i9O+nSQz6j5dv3Du6eaResrtK7tT1MFDNhcg2cgjNW64VLXQdFXYXE1OYsw +yAuXUnHNzWFhinyf80qeh2046YR21dlG8voIDQH4fGG5GmWLyu7glsWYVwQQ36VA +TTxPmAoaqUTl8A7cnlJpAo+BJ00mS/9DwJ7pkgGC7dYOhJzWlI7lPqzEfmJ+o8pj +CJlLIuqsn0vcCZQlmqCFMxK4asn+puLLnMjRLHIYEJKDNyPGHQEr2e5t4GUYZKaN +MEpXMwJd97tUamUKWeBPNIND/kOuqexe+okbOTRp34VAsK5oCpawEJckoNkK+sv0 +OrSWFOdfLBHv66p9qsrz8LQXxmN5JUBUe51SBSUo1Ul4/vGYdhuKd/8KcLw9/Al+ +HJN2hAeo3v+2fVey4hgGna7XNe8e3+E+OEQb4zpQDLkCAwEAAaOCAkswggJHMA4G +A1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYD +VR0TAQH/BAIwADAdBgNVHQ4EFgQU4PbvqCKatjx6GZMXy7v9GwykZq4wHwYDVR0j +BBgwFoAUFC6zF7dYVsuuUAlA5h+vnYsUwsYwVQYIKwYBBQUHAQEESTBHMCEGCCsG +AQUFBzABhhVodHRwOi8vcjMuby5sZW5jci5vcmcwIgYIKwYBBQUHMAKGFmh0dHA6 +Ly9yMy5pLmxlbmNyLm9yZy8wGQYDVR0RBBIwEIIOd2ViZGlveGlkZS5jb20wTAYD +VR0gBEUwQzAIBgZngQwBAgEwNwYLKwYBBAGC3xMBAQEwKDAmBggrBgEFBQcCARYa +aHR0cDovL2Nwcy5sZXRzZW5jcnlwdC5vcmcwggEGBgorBgEEAdZ5AgQCBIH3BIH0 +APIAdwBc3EOS/uarRUSxXprUVuYQN/vV+kfcoXOUsl7m9scOygAAAXw+KYGHAAAE +AwBIMEYCIQCqD6jMtHrGlE02Qh1FzFd4+qYzJTrChHmHBFIncPGQKAIhALeYk0Vf +/Lw2tX2beVlKN4/h1o8srNJv+06xkr1N6XmiAHcAfT7y+I//iFVoJMLAyp5SiXkr +xQ54CX8uapdomX4i8NcAAAF8PimBogAABAMASDBGAiEA0h883FFj1dSYKGym9+Wa +XgJRj526X7YlkhkZ5J1TjioCIQDyjMPrbo5liVi/e5b8gfDw5Fd9WNiTu1W1LKKu +UpE/qTANBgkqhkiG9w0BAQsFAAOCAQEAcx10nqp1kh2awwoqwf7Jo8Gycqx2bA2O +E2rveQ/BK9UhwvrNeEpE9SG6liMsYJKxGar0vbbBHvxzuMU00bhGjXFtUT5XuQ8q +FcU0OdycyZj8fjZmUNsJr82l8HvfJ50jfxFORTgj8Ln5MWVUFlbl0nD+06l28sDc +V+r/B4394fkoMsKXtiTA4/ZeOD1tHNsdxQ7sNQtEfqCG0wFCYHK3rs7XTZ1K0F3c +M051JShko1UKP/k5blrendOwVRwLtq+9pavGnJBeqNIVgugTER/IHlp4427WyhdY +KYjKoytW+XQyWqxU/Mh/O4rxkD8cZaE+FdZpP67VZ185AuZMbn+LcQ== +-----END CERTIFICATE----- + +-----BEGIN CERTIFICATE----- +MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAw +WhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3Mg +RW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cP +R5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdx +sxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8Zutm +NHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxg +Z3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG +/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYB +Af8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaA +FHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcw +AoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRw +Oi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQB +gt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6W +PTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wl +ikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQz +CkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BIm +lJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4 +avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2 +yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1O +yK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90Ids +hCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+ +HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6Zv +MldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqX +nLRbwHOoq7hHwg== +-----END CERTIFICATE----- + +-----BEGIN CERTIFICATE----- +MIIFYDCCBEigAwIBAgIQQAF3ITfU6UK47naqPGQKtzANBgkqhkiG9w0BAQsFADA/ +MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT +DkRTVCBSb290IENBIFgzMB4XDTIxMDEyMDE5MTQwM1oXDTI0MDkzMDE4MTQwM1ow +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCt6CRz9BQ385ueK1coHIe+3LffOJCMbjzmV6B493XC +ov71am72AE8o295ohmxEk7axY/0UEmu/H9LqMZshftEzPLpI9d1537O4/xLxIZpL +wYqGcWlKZmZsj348cL+tKSIG8+TA5oCu4kuPt5l+lAOf00eXfJlII1PoOK5PCm+D +LtFJV4yAdLbaL9A4jXsDcCEbdfIwPPqPrt3aY6vrFk/CjhFLfs8L6P+1dy70sntK +4EwSJQxwjQMpoOFTJOwT2e4ZvxCzSow/iaNhUd6shweU9GNx7C7ib1uYgeGJXDR5 +bHbvO5BieebbpJovJsXQEOEO3tkQjhb7t/eo98flAgeYjzYIlefiN5YNNnWe+w5y +sR2bvAP5SQXYgd0FtCrWQemsAXaVCg/Y39W9Eh81LygXbNKYwagJZHduRze6zqxZ +Xmidf3LWicUGQSk+WT7dJvUkyRGnWqNMQB9GoZm1pzpRboY7nn1ypxIFeFntPlF4 +FQsDj43QLwWyPntKHEtzBRL8xurgUBN8Q5N0s8p0544fAQjQMNRbcTa0B7rBMDBc +SLeCO5imfWCKoqMpgsy6vYMEG6KDA0Gh1gXxG8K28Kh8hjtGqEgqiNx2mna/H2ql +PRmP6zjzZN7IKw0KKP/32+IVQtQi0Cdd4Xn+GOdwiK1O5tmLOsbdJ1Fu/7xk9TND +TwIDAQABo4IBRjCCAUIwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +SwYIKwYBBQUHAQEEPzA9MDsGCCsGAQUFBzAChi9odHRwOi8vYXBwcy5pZGVudHJ1 +c3QuY29tL3Jvb3RzL2RzdHJvb3RjYXgzLnA3YzAfBgNVHSMEGDAWgBTEp7Gkeyxx ++tvhS5B1/8QVYIWJEDBUBgNVHSAETTBLMAgGBmeBDAECATA/BgsrBgEEAYLfEwEB +ATAwMC4GCCsGAQUFBwIBFiJodHRwOi8vY3BzLnJvb3QteDEubGV0c2VuY3J5cHQu +b3JnMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmwuaWRlbnRydXN0LmNvbS9E +U1RST09UQ0FYM0NSTC5jcmwwHQYDVR0OBBYEFHm0WeZ7tuXkAXOACIjIGlj26Ztu +MA0GCSqGSIb3DQEBCwUAA4IBAQAKcwBslm7/DlLQrt2M51oGrS+o44+/yQoDFVDC +5WxCu2+b9LRPwkSICHXM6webFGJueN7sJ7o5XPWioW5WlHAQU7G75K/QosMrAdSW +9MUgNTP52GE24HGNtLi1qoJFlcDyqSMo59ahy2cI2qBDLKobkx/J3vWraV0T9VuG +WCLKTVXkcGdtwlfFRjlBz4pYg1htmf5X6DYO8A4jqv2Il9DjXA6USbW1FzXSLr9O +he8Y4IWS6wY7bCkjCWDcRQJMEhg76fsO3txE+FiYruq9RUWhiF1myv4Q6W+CyBFC +Dfvp7OOGAN6dEOM4+qR9sdjoSYKEBpsr6GtPAQw4dy753ec5 +-----END CERTIFICATE----- diff --git a/spec/frontend/reports/grouped_test_report/grouped_test_reports_app_spec.js b/spec/frontend/reports/grouped_test_report/grouped_test_reports_app_spec.js index 0f7c2559e8b..c60c1f7b63c 100644 --- a/spec/frontend/reports/grouped_test_report/grouped_test_reports_app_spec.js +++ b/spec/frontend/reports/grouped_test_report/grouped_test_reports_app_spec.js @@ -24,7 +24,7 @@ describe('Grouped test reports app', () => { let wrapper; let mockStore; - const mountComponent = ({ props = { pipelinePath }, glFeatures = {} } = {}) => { + const mountComponent = ({ props = { pipelinePath } } = {}) => { wrapper = mount(GroupedTestReportsApp, { store: mockStore, localVue, @@ -34,9 +34,6 @@ describe('Grouped test reports app', () => { pipelinePath, ...props, }, - provide: { - glFeatures, - }, }); }; @@ -114,8 +111,8 @@ describe('Grouped test reports app', () => { setReports(newFailedTestReports); }); - it('tracks service ping metric when enabled', () => { - mountComponent({ glFeatures: { usageDataITestingSummaryWidgetTotal: true } }); + it('tracks service ping metric', () => { + mountComponent(); findExpandButton().trigger('click'); expect(Api.trackRedisHllUserEvent).toHaveBeenCalledTimes(1); @@ -123,7 +120,7 @@ describe('Grouped test reports app', () => { }); it('only tracks the first expansion', () => { - mountComponent({ glFeatures: { usageDataITestingSummaryWidgetTotal: true } }); + mountComponent(); const expandButton = findExpandButton(); expandButton.trigger('click'); expandButton.trigger('click'); @@ -131,13 +128,6 @@ describe('Grouped test reports app', () => { expect(Api.trackRedisHllUserEvent).toHaveBeenCalledTimes(1); }); - - it('does not track service ping metric when disabled', () => { - mountComponent({ glFeatures: { usageDataITestingSummaryWidgetTotal: false } }); - findExpandButton().trigger('click'); - - expect(Api.trackRedisHllUserEvent).not.toHaveBeenCalled(); - }); }); describe('with new failed result', () => { diff --git a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb index 56cf58dcf92..0a1f04ed793 100644 --- a/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb +++ b/spec/lib/gitlab/email/hook/smime_signature_interceptor_spec.rb @@ -14,15 +14,15 @@ RSpec.describe Gitlab::Email::Hook::SmimeSignatureInterceptor do end let(:root_certificate) do - Gitlab::Email::Smime::Certificate.new(@root_ca[:key], @root_ca[:cert]) + Gitlab::X509::Certificate.new(@root_ca[:key], @root_ca[:cert]) end let(:intermediate_certificate) do - Gitlab::Email::Smime::Certificate.new(@intermediate_ca[:key], @intermediate_ca[:cert]) + Gitlab::X509::Certificate.new(@intermediate_ca[:key], @intermediate_ca[:cert]) end let(:certificate) do - Gitlab::Email::Smime::Certificate.new(@cert[:key], @cert[:cert], [intermediate_certificate.cert]) + Gitlab::X509::Certificate.new(@cert[:key], @cert[:cert], [intermediate_certificate.cert]) end let(:mail_body) { "signed hello with Unicode €áø and\r\n newlines\r\n" } @@ -36,7 +36,7 @@ RSpec.describe Gitlab::Email::Hook::SmimeSignatureInterceptor do end before do - allow(Gitlab::Email::Smime::Certificate).to receive_messages(from_files: certificate) + allow(Gitlab::X509::Certificate).to receive_messages(from_files: certificate) Mail.register_interceptor(described_class) mail.deliver_now diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb index 4ce51e37685..00beacd4b35 100644 --- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb @@ -41,14 +41,40 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder ) end - it 'returns records in correct order' do + let(:all_records) do all_records = [] iterator.each_batch(of: batch_size) do |records| all_records.concat(records) end + all_records + end + it 'returns records in correct order' do expect(all_records).to eq(expected_order) end + + context 'when not passing the finder query' do + before do + in_operator_optimization_options.delete(:finder_query) + end + + it 'returns records in correct order' do + expect(all_records).to eq(expected_order) + end + + it 'loads only the order by column' do + order_by_attribute_names = iterator + .send(:order) + .column_definitions + .map(&:attribute_name) + .map(&:to_s) + + record = all_records.first + loaded_attributes = record.attributes.keys - ['time_estimate'] # time_estimate is always present (has default value) + + expect(loaded_attributes).to eq(order_by_attribute_names) + end + end end context 'when ordering by issues.id DESC' do diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb new file mode 100644 index 00000000000..fe95d5406dd --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::OrderValuesLoaderStrategy do + let(:model) { Project } + + let(:keyset_scope) do + scope, _ = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build( + Project.order(:created_at, :id) + ) + + scope + end + + let(:keyset_order) do + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(keyset_scope) + end + + let(:order_by_columns) do + Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumns.new(keyset_order.column_definitions, model.arel_table) + end + + subject(:strategy) { described_class.new(model, order_by_columns) } + + describe '#initializer_columns' do + it 'returns NULLs for each ORDER BY columns' do + expect(strategy.initializer_columns).to eq([ + 'NULL::timestamp without time zone AS created_at', + 'NULL::integer AS id' + ]) + end + end +end diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb new file mode 100644 index 00000000000..5180403b493 --- /dev/null +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::RecordLoaderStrategy do + let(:finder_query) { -> (created_at_value, id_value) { Project.where(Project.arel_table[:id].eq(id_value)) } } + let(:model) { Project } + + let(:keyset_scope) do + scope, _ = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build( + Project.order(:created_at, :id) + ) + + scope + end + + let(:keyset_order) do + Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(keyset_scope) + end + + let(:order_by_columns) do + Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumns.new(keyset_order.column_definitions, model.arel_table) + end + + subject(:strategy) { described_class.new(finder_query, model, order_by_columns) } + + describe '#initializer_columns' do + # Explanation: + # > SELECT NULL::projects AS records + # + # The query returns one row and one column. The column may contain a full project row. + # In this particular case the row is NULL. + it 'returns a NULL table row as the result column' do + expect(strategy.initializer_columns).to eq(["NULL::projects AS records"]) + end + end + + describe '#columns' do + # Explanation: + # > SELECT (SELECT projects FROM projects limit 1) + # + # Selects one row from the database and collapses it into one column. + # + # Side note: Due to the type casts, columns and initializer_columns can be also UNION-ed: + # SELECT * FROM ( + # ( + # SELECT NULL::projects AS records + # UNION + # SELECT (SELECT projects FROM projects limit 1) + # ) + # ) as records + it 'uses the finder query to load the row in the result column' do + expected_loader_query = <<~SQL + (SELECT projects FROM "projects" WHERE "projects"."id" = recursive_keyset_cte.projects_id_array[position] LIMIT 1) + SQL + + expect(strategy.columns).to eq([expected_loader_query.chomp]) + end + end +end diff --git a/spec/lib/gitlab/email/smime/certificate_spec.rb b/spec/lib/gitlab/x509/certificate_spec.rb similarity index 98% rename from spec/lib/gitlab/email/smime/certificate_spec.rb rename to spec/lib/gitlab/x509/certificate_spec.rb index f7bb933e348..a5b192dd051 100644 --- a/spec/lib/gitlab/email/smime/certificate_spec.rb +++ b/spec/lib/gitlab/x509/certificate_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Email::Smime::Certificate do +RSpec.describe Gitlab::X509::Certificate do include SmimeHelper # cert generation is an expensive operation and they are used read-only, diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index 7b997f0d4e1..b584f011972 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -287,6 +287,19 @@ RSpec.describe PagesDomain do it { is_expected.to be_truthy } end + + # The LetsEncrypt DST Root CA X3 expired on 2021-09-30, but the + # cross-sign in ISRG Root X1 enables it to function provided a chain + # of trust can be established with the system store. See: + # + # 1. https://community.letsencrypt.org/t/production-chain-changes/150739 + # 2. https://letsencrypt.org/2020/12/21/extending-android-compatibility.html + # 3. https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ + context 'with a LetsEncrypt bundle with an expired DST Root CA X3' do + let(:domain) { build(:pages_domain, :letsencrypt_expired_x3_root) } + + it { is_expected.to be_truthy } + end end describe '#expired?' do