Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -235,6 +235,8 @@ coverage-frontend:
     - *yarn-install
   script:
     - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js"
+    # Removing the individual coverage results, as we just merged them.
+    - rm -r coverage-frontend/jest-*
   coverage: '/^Statements\s*:\s*?(\d+(?:\.\d+)?)%/'
   artifacts:
     name: coverage-frontend

.gitlab/ci/pages.gitlab-ci.yml

@@ -14,7 +14,6 @@ pages:
     - mv coverage/ public/coverage-ruby/ || true
     - mv coverage-frontend/ public/coverage-frontend/ || true
     - mv coverage-javascript/ public/coverage-javascript/ || true
-    - mv webpack-report/ public/webpack-report/ || true
     - cp .public/assets/application-*.css public/application.css || true
     - cp .public/assets/application-*.css.gz public/application.css.gz || true
   artifacts:

.gitlab/ci/review.gitlab-ci.yml

@@ -38,7 +38,7 @@ review-build-cng:
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
     # When the job is manual, review-deploy is also manual and we don't want people
     # to have to manually start the jobs in sequence, so we do it for them.
-    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-deploy"'
+    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-deploy"'
 
 .review-workflow-base:
   extends:
@@ -78,8 +78,8 @@ review-deploy:
     - disable_sign_ups || (delete_release && exit 1)
     # When the job is manual, review-qa-smoke is also manual and we don't want people
     # to have to manually start the jobs in sequence, so we do it for them.
-    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-qa-smoke"'
-    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-performance"'
+    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-qa-smoke"'
+    - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-performance"'
   after_script:
     # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan.
     # Set DAST_RUN to true when jobs are manually scheduled.

app/graphql/types/repository_type.rb

@@ -10,7 +10,7 @@ module Types
           description: 'Default branch of the repository'
     field :empty, GraphQL::BOOLEAN_TYPE, null: false, method: :empty?, calls_gitaly: true,
           description: 'Indicates repository has no visible content'
-    field :exists, GraphQL::BOOLEAN_TYPE, null: false, method: :exists?,
+    field :exists, GraphQL::BOOLEAN_TYPE, null: false, method: :exists?, calls_gitaly: true,
           description: 'Indicates a corresponding Git repository exists on disk'
     field :tree, Types::Tree::TreeType, null: true, resolver: Resolvers::TreeResolver, calls_gitaly: true,
           description: 'Tree of the repository'

app/views/devise/shared/_footer.html.haml

@@ -0,0 +1,8 @@
+%hr.footer-fixed
+.container.footer-container
+  .footer-links
+    - unless public_visibility_restricted?
+      = link_to _("Explore"), explore_root_path
+      = link_to _("Help"), help_path
+    = link_to _("About GitLab"), "https://about.gitlab.com/"
+= footer_message

app/views/layouts/devise.html.haml

@@ -38,12 +38,4 @@
             .col-sm-5.order-1.order-sm-12.new-session-forms-container
               = yield
 
-      %hr.footer-fixed
-      .container.footer-container
-        .footer-links
-          - if !public_visibility_restricted?
-            = link_to _("Explore"), explore_root_path
-            = link_to _("Help"), help_path
-          = link_to _("About GitLab"), "https://about.gitlab.com/"
-
-      = footer_message
+      = render 'devise/shared/footer', footer_message: footer_message

app/views/layouts/devise_empty.html.haml

@@ -1,5 +1,5 @@
 !!! 5
-%html{ lang: "en", class: system_message_class }
+%html.devise-layout-html{ lang: "en", class: system_message_class }
   = render "layouts/head"
   %body.ui-indigo.login-page.application.navless{ class: "#{client_class_list}" }
     = header_message
@@ -11,11 +11,4 @@
         = render "layouts/flash"
         = yield
 
-    %hr
-    .container
-      .footer-links
-        - if !public_visibility_restricted?
-          = link_to _("Explore"), explore_root_path
-          = link_to _("Help"), help_path
-        = link_to _("About GitLab"), "https://about.gitlab.com/"
-    = footer_message
+    = render 'devise/shared/footer', footer_message: footer_message

changelogs/unreleased/georgekoltsov-limit-extractors-loaders-to-one.yml

@@ -0,0 +1,5 @@
+---
+title: Limit Group Migration extractors and loaders to 1 per pipeline
+merge_request: 50951
+author:
+type: changed

changelogs/unreleased/ld-add-calls-gitaly-to-fields.yml

@@ -0,0 +1,6 @@
+---
+title: Increase the complexity score of GraphQL MergeRequest#approved, MergeRequest#approvalsLeft
+  and Repository#exists fields as they can call Gitaly
+merge_request: 47039
+author:
+type: changed

danger/frozen_string/Dangerfile

@@ -1,15 +1,34 @@
 # frozen_string_literal: true
 
 FILE_EXTENSION = ".rb"
-MAGIC_COMMENT = "# frozen_string_literal: true"
+FROZEN_STRING_MAGIC_COMMENT = "# frozen_string_literal: true"
+SHEBANG_COMMENT = "#!"
 
 def get_files_with_no_magic_comment(files)
-  files.select do |file|
-    file.end_with?(FILE_EXTENSION) &&
-      !File.open(file, &:gets)&.start_with?(MAGIC_COMMENT)
+  files.select do |path|
+    path.end_with?(FILE_EXTENSION) &&
+      file_has_frozen_string_magic_comment?(path)
   end
 end
 
+def file_has_frozen_string_magic_comment?(path)
+  File.open(path) do |file|
+    first_line = file.gets
+
+    line_has_frozen_string_magic_comment?(first_line) ||
+      (line_has_shebang?(first_line) &&
+        line_has_frozen_string_magic_comment?(file.gets))
+  end
+end
+
+def line_has_frozen_string_magic_comment?(line)
+  line&.start_with?(FROZEN_STRING_MAGIC_COMMENT)
+end
+
+def line_has_shebang?(line)
+  line&.start_with?(SHEBANG_COMMENT)
+end
+
 files_to_fix = get_files_with_no_magic_comment(git.added_files)
 
 if files_to_fix.any?
@@ -20,7 +39,7 @@ if files_to_fix.any?
     markdown(<<~MARKDOWN)
       ## Enable Frozen String Literal
 
-      The following files should have `#{MAGIC_COMMENT}` on the first line:
+      The following files should have `#{FROZEN_STRING_MAGIC_COMMENT}` on the first line:
 
       * #{files_to_fix.map { |path| "`#{path}`" }.join("\n* ")}
     MARKDOWN

doc/user/group/saml_sso/index.md

@@ -80,10 +80,13 @@ Please note that the certificate [fingerprint algorithm](#additional-providers-a
 
 - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8.
 - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI.
+- [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/292811) in GitLab 13.8, with an updated timeout experience.
 
 With this option enabled, users must go through your group's GitLab single sign-on URL. They may also be added via SCIM, if configured. Users can't be added manually, and may only access project/group resources via the UI by signing in through the SSO URL.
 
-However, users are not prompted to sign in through SSO on each visit. GitLab checks whether a user has authenticated through SSO, and only prompts the user to sign in via SSO if the session has expired.
+However, users are not prompted to sign in through SSO on each visit. GitLab checks whether a user
+has authenticated through SSO. If it's been more than 7 days since the last sign-in, GitLab
+prompts the user to sign in again through SSO.
 You can see more information about how long a session is valid in our [user profile documentation](../../profile/#why-do-i-keep-getting-signed-out).
 
 We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/-/issues/9152).

lib/bulk_imports/pipeline.rb

@@ -10,16 +10,16 @@ module BulkImports
 
       private
 
-      def extractors
-        @extractors ||= self.class.extractors.map(&method(:instantiate))
+      def extractor
+        @extractor ||= instantiate(self.class.get_extractor)
       end
 
       def transformers
         @transformers ||= self.class.transformers.map(&method(:instantiate))
       end
 
-      def loaders
-        @loaders ||= self.class.loaders.map(&method(:instantiate))
+      def loader
+        @loaders ||= instantiate(self.class.get_loader)
       end
 
       def after_run
@@ -41,7 +41,7 @@ module BulkImports
 
     class_methods do
       def extractor(klass, options = nil)
-        add_attribute(:extractors, klass, options)
+        class_attributes[:extractor] = { klass: klass, options: options }
       end
 
       def transformer(klass, options = nil)
@@ -49,23 +49,23 @@ module BulkImports
       end
 
       def loader(klass, options = nil)
-        add_attribute(:loaders, klass, options)
+        class_attributes[:loader] = { klass: klass, options: options }
       end
 
       def after_run(&block)
         class_attributes[:after_run] = block
       end
 
-      def extractors
-        class_attributes[:extractors]
+      def get_extractor
+        class_attributes[:extractor]
       end
 
       def transformers
         class_attributes[:transformers]
       end
 
-      def loaders
-        class_attributes[:loaders]
+      def get_loader
+        class_attributes[:loader]
       end
 
       def after_run_callback

lib/bulk_imports/pipeline/runner.rb

@@ -12,25 +12,15 @@ module BulkImports
 
         info(context, message: 'Pipeline started', pipeline_class: pipeline)
 
-        extractors.each do |extractor|
-          data = run_pipeline_step(:extractor, extractor.class.name, context) do
-            extractor.extract(context)
+        Array.wrap(extracted_data_from(context)).each do |entry|
+          transformers.each do |transformer|
+            entry = run_pipeline_step(:transformer, transformer.class.name, context) do
+              transformer.transform(context, entry)
+            end
           end
 
-          if data && data.respond_to?(:each)
-            data.each do |entry|
-              transformers.each do |transformer|
-                entry = run_pipeline_step(:transformer, transformer.class.name, context) do
-                  transformer.transform(context, entry)
-                end
-              end
-
-              loaders.each do |loader|
-                run_pipeline_step(:loader, loader.class.name, context) do
-                  loader.load(context, entry)
-                end
-              end
-            end
+          run_pipeline_step(:loader, loader.class.name, context) do
+            loader.load(context, entry)
           end
         end
 
@@ -55,6 +45,12 @@ module BulkImports
         mark_as_failed(context) if abort_on_failure?
       end
 
+      def extracted_data_from(context)
+        run_pipeline_step(:extractor, extractor.class.name, context) do
+          extractor.extract(context)
+        end
+      end
+
       def mark_as_failed(context)
         warn(context, message: 'Pipeline failed', pipeline_class: pipeline)
 

locale/gitlab.pot

@@ -23136,6 +23136,9 @@ msgstr ""
 msgid "Real-time features"
 msgstr ""
 
+msgid "Reauthenticating with SAML provider."
+msgstr ""
+
 msgid "Rebase"
 msgstr ""
 

scripts/api/play_job.rb

@@ -14,7 +14,6 @@ class PlayJob
   }.freeze
 
   def initialize(options)
-    @project = options.delete(:project)
     @options = options
 
     Gitlab.configure do |config|
@@ -24,14 +23,18 @@ class PlayJob
   end
 
   def execute
-    job = JobFinder.new(project, options.slice(:api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute
+    job = JobFinder.new(options.slice(:project, :api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute
 
     Gitlab.job_play(project, job.id)
   end
 
   private
 
-  attr_reader :project, :options
+  attr_reader :options
+
+  def project
+    options[:project]
+  end
 end
 
 if $0 == __FILE__

scripts/rspec_helpers.sh

@@ -7,14 +7,14 @@ function retrieve_tests_metadata() {
   local test_metadata_job_id
 
   # Ruby
-  test_metadata_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
+  test_metadata_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
 
   if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
   fi
 
   if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
   fi
 }
 
@@ -42,10 +42,10 @@ function retrieve_tests_mapping() {
   local project_path="gitlab-org/gitlab"
   local test_metadata_with_mapping_job_id
 
-  test_metadata_with_mapping_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
+  test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
 
   if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
-   (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+   (scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
   fi
 
   scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"

scripts/utils.sh

@@ -89,12 +89,12 @@ function echosuccess() {
 
 function fail_pipeline_early() {
   local dont_interrupt_me_job_id
-  dont_interrupt_me_job_id=$(scripts/api/get_job_id --job-query "scope=success" --job-name "dont-interrupt-me")
+  dont_interrupt_me_job_id=$(scripts/api/get_job_id.rb --job-query "scope=success" --job-name "dont-interrupt-me")
 
   if [[ -n "${dont_interrupt_me_job_id}" ]]; then
     echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}"
   else
     echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast."
-    scripts/api/cancel_pipeline
+    scripts/api/cancel_pipeline.rb
   fi
 }

spec/graphql/types/merge_request_type_spec.rb

@@ -32,14 +32,7 @@ RSpec.describe GitlabSchema.types['MergeRequest'] do
       has_ci mergeable commits_without_merge_commits squash security_auto_fix default_squash_commit_message
     ]
 
-    if Gitlab.ee?
-      expected_fields << 'approved'
-      expected_fields << 'approvals_left'
-      expected_fields << 'approvals_required'
-      expected_fields << 'merge_trains_count'
-    end
-
-    expect(described_class).to have_graphql_fields(*expected_fields)
+    expect(described_class).to have_graphql_fields(*expected_fields).at_least
   end
 
   describe '#pipelines' do

spec/graphql/types/repository_type_spec.rb

@@ -10,4 +10,6 @@ RSpec.describe GitlabSchema.types['Repository'] do
   specify { expect(described_class).to have_graphql_field(:root_ref) }
 
   specify { expect(described_class).to have_graphql_field(:tree) }
+
+  specify { expect(described_class).to have_graphql_field(:exists, calls_gitaly?: true, complexity: 2) }
 end

spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb

@@ -75,13 +75,11 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do
     it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
 
     it 'has extractors' do
-      expect(described_class.extractors)
-        .to contain_exactly(
-          {
-            klass: BulkImports::Common::Extractors::GraphqlExtractor,
-            options: {
-              query: BulkImports::Groups::Graphql::GetGroupQuery
-            }
+      expect(described_class.get_extractor)
+        .to eq(
+          klass: BulkImports::Common::Extractors::GraphqlExtractor,
+          options: {
+            query: BulkImports::Groups::Graphql::GetGroupQuery
           }
         )
     end
@@ -97,9 +95,7 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do
     end
 
     it 'has loaders' do
-      expect(described_class.loaders).to contain_exactly({
-        klass: BulkImports::Groups::Loaders::GroupLoader, options: nil
-      })
+      expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::GroupLoader, options: nil)
     end
   end
 end

spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb

@@ -58,10 +58,7 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
     it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
 
     it 'has extractors' do
-      expect(described_class.extractors).to contain_exactly(
-        klass: BulkImports::Groups::Extractors::SubgroupsExtractor,
-        options: nil
-      )
+      expect(described_class.get_extractor).to eq(klass: BulkImports::Groups::Extractors::SubgroupsExtractor, options: nil)
     end
 
     it 'has transformers' do
@@ -72,10 +69,7 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
     end
 
     it 'has loaders' do
-      expect(described_class.loaders).to contain_exactly(
-        klass: BulkImports::Common::Loaders::EntityLoader,
-        options: nil
-      )
+      expect(described_class.get_loader).to eq(klass: BulkImports::Common::Loaders::EntityLoader, options: nil)
     end
   end
 end

spec/lib/bulk_imports/pipeline_spec.rb

@@ -24,9 +24,9 @@ RSpec.describe BulkImports::Pipeline do
 
     describe 'getters' do
       it 'retrieves class attributes' do
-        expect(BulkImports::MyPipeline.extractors).to contain_exactly({ klass: BulkImports::Extractor, options: { foo: :bar } })
+        expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: BulkImports::Extractor, options: { foo: :bar } })
         expect(BulkImports::MyPipeline.transformers).to contain_exactly({ klass: BulkImports::Transformer, options: { foo: :bar } })
-        expect(BulkImports::MyPipeline.loaders).to contain_exactly({ klass: BulkImports::Loader, options: { foo: :bar } })
+        expect(BulkImports::MyPipeline.get_loader).to eq({ klass: BulkImports::Loader, options: { foo: :bar } })
         expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true)
       end
     end
@@ -41,20 +41,14 @@ RSpec.describe BulkImports::Pipeline do
         BulkImports::MyPipeline.loader(klass, options)
         BulkImports::MyPipeline.abort_on_failure!
 
-        expect(BulkImports::MyPipeline.extractors)
-          .to contain_exactly(
-            { klass: BulkImports::Extractor, options: { foo: :bar } },
-            { klass: klass, options: options })
+        expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: klass, options: options })
 
         expect(BulkImports::MyPipeline.transformers)
           .to contain_exactly(
             { klass: BulkImports::Transformer, options: { foo: :bar } },
             { klass: klass, options: options })
 
-        expect(BulkImports::MyPipeline.loaders)
-          .to contain_exactly(
-            { klass: BulkImports::Loader, options: { foo: :bar } },
-            { klass: klass, options: options })
+        expect(BulkImports::MyPipeline.get_loader).to eq({ klass: klass, options: options })
 
         expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true)
       end