diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 14eeebb9db9..a43be817ce2 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,7 +2,7 @@ extends: - .default-tags - .default-retry - - .only-docs-changes + - .only:changes-docs only: refs: - merge_requests @@ -50,7 +50,7 @@ docs lint: - .default-tags - .default-retry - .default-only - - .only-docs-changes + - .only:changes-docs image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" stage: test dependencies: [] @@ -76,7 +76,7 @@ graphql-docs-verify: - .default-cache - .default-only - .default-before_script - - .only-graphql-changes + - .only:changes-graphql variables: SETUP_DB: "false" stage: test diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 2f457bc0ee2..0b72461a9fd 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -12,7 +12,7 @@ - .default-only - .default-before_script - .assets-compile-cache - - .only-code-qa-changes + - .only:changes-code-backstage-qa image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 stage: test dependencies: ["setup-test-env"] @@ -73,7 +73,7 @@ gitlab:assets:compile pull-cache: - .default-only - .default-before_script - .assets-compile-cache - - .only-code-qa-changes + - .only:changes-code-backstage-qa - .use-pg9 stage: prepare script: @@ -128,7 +128,7 @@ compile-assets pull-cache foss: - .default-cache - .default-only - .default-before_script - - .only-code-changes + - .only:changes-code-backstage - .use-pg9 stage: test needs: ["setup-test-env", "compile-assets pull-cache"] @@ -205,7 +205,7 @@ jest-foss: - .default-retry - .default-cache - .default-only - - .only-code-changes + - .only:changes-code-backstage stage: test dependencies: [] cache: @@ -238,7 +238,7 @@ webpack-dev-server: - .default-retry - .default-cache - .default-only - - .only-code-changes + - .only:changes-code-backstage stage: test needs: ["setup-test-env", "compile-assets pull-cache"] dependencies: ["setup-test-env", "compile-assets pull-cache"] diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index fc9b00b5d3c..743f4ec3390 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -40,14 +40,64 @@ - merge_requests - tags -.only-code-changes: +.code-patterns: &code-patterns + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + +.backstage-patterns: &backstage-patterns + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" + - "doc/README.md" # Some RSpec test rely on this file + +.qa-patterns: &qa-patterns + - ".dockerignore" + - "qa/**/*" + +.docs-patterns: &docs-patterns + - ".gitlab/route-map.yml" + - "doc/**/*" + - ".markdownlint.json" + +.graphql-patterns: &graphql-patterns + - "{,ee/}app/graphql/**/*" + - "{,ee/}lib/gitlab/graphql/**/*" + +.only:changes-code: + only: + changes: *code-patterns + +.only:changes-qa: + only: + changes: *qa-patterns + +.only:changes-docs: + only: + changes: *docs-patterns + +.only:changes-graphql: + only: + changes: *graphql-patterns + +.only:changes-code-backstage: only: changes: - ".gitlab/ci/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - ".csscomb.json" - - "Dangerfile" - "Dockerfile.assets" - "*_VERSION" - "Gemfile{,.lock}" @@ -55,36 +105,41 @@ - "{babel.config,jest.config}.js" - "config.ru" - "{package.json,yarn.lock}" - - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" - "doc/README.md" # Some RSpec test rely on this file -.only-qa-changes: +.only:changes-code-qa: only: changes: + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + # QA changes - ".dockerignore" - "qa/**/*" -.only-docs-changes: - only: - changes: - - ".gitlab/route-map.yml" - - "doc/**/*" - - ".markdownlint.json" - -.only-graphql-changes: - only: - changes: - - "{,ee/}app/graphql/**/*" - - "{,ee/}lib/gitlab/graphql/**/*" - -.only-code-qa-changes: +.only:changes-code-backstage-qa: only: changes: - ".gitlab/ci/**/*" - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - ".csscomb.json" - - "Dangerfile" - "Dockerfile.assets" - "*_VERSION" - "Gemfile{,.lock}" @@ -92,8 +147,15 @@ - "{babel.config,jest.config}.js" - "config.ru" - "{package.json,yarn.lock}" - - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" - "doc/README.md" # Some RSpec test rely on this file + # QA changes - ".dockerignore" - "qa/**/*" diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 93bf87b24b2..ba14024df34 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -5,7 +5,7 @@ - .default-cache - .default-only - .default-before_script - - .only-code-changes + - .only:changes-code memory-static: extends: .only-code-memory-job-base diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index a30772d5664..a9099d140ff 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -4,7 +4,7 @@ pages: - .default-retry - .default-cache - .default-only - - .only-code-qa-changes + - .only:changes-code-backstage-qa only: refs: - master diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 1194948a76f..ef407081d50 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -3,7 +3,7 @@ - .default-tags - .default-retry - .default-only - - .only-code-qa-changes + - .only:changes-code-qa stage: test dependencies: [] cache: @@ -47,7 +47,7 @@ qa:selectors-foss: package-and-qa-manual: extends: - .package-and-qa-base - - .only-code-changes + - .only:changes-code except: refs: - master @@ -58,7 +58,7 @@ package-and-qa-manual: package-and-qa: extends: - .package-and-qa-base - - .only-qa-changes + - .only:changes-qa except: refs: - master @@ -69,6 +69,6 @@ package-and-qa: schedule:package-and-qa: extends: - .package-and-qa-base - - .only-code-qa-changes + - .only:changes-code-qa - .only-canonical-schedules needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index bf478b68765..32cada6ea40 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -22,7 +22,7 @@ - .default-cache - .default-only - .default-before_script - - .only-code-changes + - .only:changes-code-backstage .only-code-qa-rails-job-base: extends: @@ -31,7 +31,7 @@ - .default-cache - .default-only - .default-before_script - - .only-code-qa-changes + - .only:changes-code-backstage-qa setup-test-env: extends: @@ -251,7 +251,7 @@ static-analysis: downtime_check: extends: - .rake-exec - - .only-code-changes + - .only:changes-code-backstage except: refs: - master diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 16c3f0e4f8c..fbb7826b6f2 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -11,7 +11,7 @@ code_quality: extends: - .default-retry - .default-only - - .only-code-changes + - .only:changes-code-backstage stage: test image: docker:stable allow_failure: true @@ -50,7 +50,7 @@ sast: extends: - .default-retry - .default-only - - .only-code-changes + - .only:changes-code-backstage-qa stage: test image: docker:stable variables: @@ -132,7 +132,7 @@ dependency_scanning: extends: - .default-retry - .default-only - - .only-code-changes + - .only:changes-code-backstage-qa stage: test image: docker:stable variables: @@ -195,7 +195,7 @@ dast: extends: - .default-retry - .default-only - - .only-code-qa-changes + - .only:changes-code-qa - .only-review stage: qa needs: ["review-deploy"] diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index a71c4ea2f1d..539ae8a2512 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -23,7 +23,7 @@ build-qa-image: extends: - .review-docker - - .only-code-qa-changes + - .only:changes-code-qa only: variables: - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" @@ -40,7 +40,7 @@ build-qa-image: - .default-tags - .default-retry - .default-only - - .only-code-qa-changes + - .only:changes-code-qa stage: prepare image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base allow_failure: true @@ -66,7 +66,7 @@ manual:review-cleanup: .review-build-cng-base: extends: - .default-only - - .only-code-qa-changes + - .only:changes-code-qa - .except-deploys image: ruby:2.6-alpine stage: review-prepare @@ -95,7 +95,7 @@ schedule:review-build-cng: - .default-tags - .default-retry - .default-only - - .only-code-qa-changes + - .only:changes-code-qa - .except-deploys stage: review image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base @@ -189,7 +189,7 @@ review-cleanup-failed-deployment: extends: - .review-docker - .only-review - - .only-code-qa-changes + - .only:changes-code-qa stage: qa allow_failure: true variables: @@ -236,7 +236,7 @@ review-qa-all: .review-performance-base: extends: - .review-docker - - .only-code-qa-changes + - .only:changes-code-qa stage: qa allow_failure: true before_script: @@ -288,7 +288,7 @@ parallel-spec-reports: extends: - .default-tags - .default-only - - .only-code-qa-changes + - .only:changes-code-qa - .only-review - .except-deploys image: ruby:2.6-alpine diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 861f3f1af5b..605f431b5ff 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -6,7 +6,7 @@ cache gems: - .default-retry - .default-cache - .default-before_script - - .only-code-qa-changes + - .only:changes-code-backstage-qa stage: test dependencies: ["setup-test-env"] needs: ["setup-test-env"] @@ -29,7 +29,7 @@ cache gems: - .default-tags - .default-retry - .default-only - - .only-code-changes + - .only:changes-code-backstage dependencies: [] gitlab_git_test: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 6a7f3157d59..21af0d373bc 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,7 +1,7 @@ .tests-metadata-state: extends: - .default-only - - .only-code-changes + - .only:changes-code-backstage variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -48,7 +48,7 @@ flaky-examples-check: - .default-tags - .default-retry - .default-only - - .only-code-changes + - .only:changes-code-backstage image: ruby:2.6-alpine stage: post-test variables: diff --git a/app/assets/javascripts/lib/graphql.js b/app/assets/javascripts/lib/graphql.js index c05db4a5c71..ca797cde913 100644 --- a/app/assets/javascripts/lib/graphql.js +++ b/app/assets/javascripts/lib/graphql.js @@ -26,7 +26,8 @@ export default (resolvers = {}, config = {}) => { createUploadLink(httpOptions), new BatchHttpLink(httpOptions), ), - cache: new InMemoryCache(config.cacheConfig), + cache: new InMemoryCache({ ...config.cacheConfig, freezeResults: true }), resolvers, + assumeImmutableResults: true, }); }; diff --git a/app/assets/stylesheets/pages/experimental_separate_sign_up.scss b/app/assets/stylesheets/pages/experimental_separate_sign_up.scss index 8b1ec1ced35..53dfdd10788 100644 --- a/app/assets/stylesheets/pages/experimental_separate_sign_up.scss +++ b/app/assets/stylesheets/pages/experimental_separate_sign_up.scss @@ -23,6 +23,7 @@ .signup-heading h2 { font-weight: $gl-font-weight-bold; + padding: 0 10px; @include media-breakpoint-down(md) { font-size: $gl-font-size-large; diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4089fcf7097..62815589c3c 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -35,6 +35,10 @@ module Ci refspecs: -> (build) { build.merge_request_ref? } }.freeze + DEFAULT_RETRIES = { + scheduler_failure: 2 + }.freeze + has_one :deployment, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id @@ -372,18 +376,25 @@ module Ci pipeline.builds.retried.where(name: self.name).count end - def retries_max - normalized_retry.fetch(:max, 0) - end - - def retry_when - normalized_retry.fetch(:when, ['always']) - end - def retry_failure? - return false if retries_max.zero? || retries_count >= retries_max + max_allowed_retries = nil + max_allowed_retries ||= options_retry_max if retry_on_reason_or_always? + max_allowed_retries ||= DEFAULT_RETRIES.fetch(failure_reason.to_sym, 0) - retry_when.include?('always') || retry_when.include?(failure_reason.to_s) + max_allowed_retries > 0 && retries_count < max_allowed_retries + end + + def options_retry_max + options_retry[:max] + end + + def options_retry_when + options_retry.fetch(:when, ['always']) + end + + def retry_on_reason_or_always? + options_retry_when.include?(failure_reason.to_s) || + options_retry_when.include?('always') end def latest? @@ -831,6 +842,13 @@ module Ci :creating end + # Consider this object to have a structural integrity problems + def doom! + update_columns( + status: :failed, + failure_reason: :data_integrity_failure) + end + private def successful_deployment_status @@ -875,8 +893,8 @@ module Ci # format, but builds created before GitLab 11.5 and saved in database still # have the old integer only format. This method returns the retry option # normalized as a hash in 11.5+ format. - def normalized_retry - strong_memoize(:normalized_retry) do + def options_retry + strong_memoize(:options_retry) do value = options&.dig(:retry) value = value.is_a?(Integer) ? { max: value } : value.to_h value.with_indifferent_access diff --git a/app/models/commit_status_enums.rb b/app/models/commit_status_enums.rb index a540e291990..2ca6d15e642 100644 --- a/app/models/commit_status_enums.rb +++ b/app/models/commit_status_enums.rb @@ -15,7 +15,9 @@ module CommitStatusEnums stale_schedule: 7, job_execution_timeout: 8, archived_failure: 9, - unmet_prerequisites: 10 + unmet_prerequisites: 10, + scheduler_failure: 11, + data_integrity_failure: 12 } end end diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb index f1182ec26f4..66ae840a619 100644 --- a/app/presenters/commit_status_presenter.rb +++ b/app/presenters/commit_status_presenter.rb @@ -11,7 +11,9 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated stale_schedule: 'Delayed job could not be executed by some reason, please try again', job_execution_timeout: 'The script exceeded the maximum execution time set for the job', archived_failure: 'The job is archived and cannot be run', - unmet_prerequisites: 'The job failed to complete prerequisite tasks' + unmet_prerequisites: 'The job failed to complete prerequisite tasks', + scheduler_failure: 'The scheduler failed to assign job to the runner, please try again or contact system administrator', + data_integrity_failure: 'There has been a structural integrity problem detected, please contact system administrator' }.freeze private_constant :CALLOUT_FAILURE_MESSAGES @@ -33,6 +35,6 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated end def unrecoverable? - script_failure? || missing_dependency_failure? || archived_failure? + script_failure? || missing_dependency_failure? || archived_failure? || scheduler_failure? || data_integrity_failure? end end diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index d8f32ff88ce..ff140444c1c 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -42,26 +42,16 @@ module Ci end builds.each do |build| - next unless runner.can_pick?(build) + result = process_build(build, params) + next unless result - begin - # In case when 2 runners try to assign the same build, second runner will be declined - # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method. - if assign_runner!(build, params) - register_success(build) + if result.valid? + register_success(result.build) - return Result.new(build, true) - end - rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError - # We are looping to find another build that is not conflicting - # It also indicates that this build can be picked and passed to runner. - # If we don't do it, basically a bunch of runners would be competing for a build - # and thus we will generate a lot of 409. This will increase - # the number of generated requests, also will reduce significantly - # how many builds can be picked by runner in a unit of time. - # In case we hit the concurrency-access lock, - # we still have to return 409 in the end, - # to make sure that this is properly handled by runner. + return result + else + # The usage of valid: is described in + # handling of ActiveRecord::StaleObjectError valid = false end end @@ -73,6 +63,35 @@ module Ci private + def process_build(build, params) + return unless runner.can_pick?(build) + + # In case when 2 runners try to assign the same build, second runner will be declined + # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method. + if assign_runner!(build, params) + Result.new(build, true) + end + rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError + # We are looping to find another build that is not conflicting + # It also indicates that this build can be picked and passed to runner. + # If we don't do it, basically a bunch of runners would be competing for a build + # and thus we will generate a lot of 409. This will increase + # the number of generated requests, also will reduce significantly + # how many builds can be picked by runner in a unit of time. + # In case we hit the concurrency-access lock, + # we still have to return 409 in the end, + # to make sure that this is properly handled by runner. + Result.new(nil, false) + rescue => ex + raise ex unless Feature.enabled?(:ci_doom_build, default_enabled: true) + + scheduler_failure!(build) + track_exception_for_build(ex, build) + + # skip, and move to next one + nil + end + def assign_runner!(build, params) build.runner_id = runner.id build.runner_session_attributes = params[:session] if params[:session].present? @@ -96,6 +115,28 @@ module Ci true end + def scheduler_failure!(build) + Gitlab::OptimisticLocking.retry_lock(build, 3) do |subject| + subject.drop!(:scheduler_failure) + end + rescue => ex + build.doom! + + # This requires extra exception, otherwise we would loose information + # why we cannot perform `scheduler_failure` + track_exception_for_build(ex, build) + end + + def track_exception_for_build(ex, build) + Gitlab::Sentry.track_acceptable_exception(ex, extra: { + build_id: build.id, + build_name: build.name, + build_stage: build.stage, + pipeline_id: build.pipeline_id, + project_id: build.project_id + }) + end + # rubocop: disable CodeReuse/ActiveRecord def builds_for_shared_runner new_builds. diff --git a/app/views/registrations/welcome.html.haml b/app/views/registrations/welcome.html.haml index 02ab974ecc0..76c4a935584 100644 --- a/app/views/registrations/welcome.html.haml +++ b/app/views/registrations/welcome.html.haml @@ -1,4 +1,4 @@ -- content_for(:page_title, _('Welcome to GitLab
%{username}!' % { username: html_escape(current_user.username) }).html_safe) +- content_for(:page_title, _('Welcome to GitLab %{username}!') % { username: current_user.username }) - max_name_length = 128 .text-center.mb-3 = _('In order to tailor your experience with GitLab
we would like to know a bit more about you.').html_safe diff --git a/changelogs/unreleased/13539-license-compliance-approval-required.yml b/changelogs/unreleased/13539-license-compliance-approval-required.yml new file mode 100644 index 00000000000..8fa91775dac --- /dev/null +++ b/changelogs/unreleased/13539-license-compliance-approval-required.yml @@ -0,0 +1,5 @@ +--- +title: Show approval required status in license compliance +merge_request: 19114 +author: +type: changed diff --git a/changelogs/unreleased/make-register-job-service-to-be-resillient.yml b/changelogs/unreleased/make-register-job-service-to-be-resillient.yml new file mode 100644 index 00000000000..7587447c252 --- /dev/null +++ b/changelogs/unreleased/make-register-job-service-to-be-resillient.yml @@ -0,0 +1,5 @@ +--- +title: Make `jobs/request` to be resillient +merge_request: 19150 +author: +type: fixed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index f6814262b7a..02d66067418 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -8,7 +8,7 @@ # If a setting requires an application restart say so in that screen. # # If you change this file in a Merge Request, please also create # # a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests. # -# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/0928cfb09f43993fd9454b0b14dbd1924b1407bc/doc/settings/gitlab.yml.md # +# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md # ######################################################################## # # diff --git a/db/migrate/20190805140353_remove_rendundant_index_from_releases.rb b/db/migrate/20190805140353_remove_rendundant_index_from_releases.rb index fc4bc1a423b..a0dcd68e8b4 100644 --- a/db/migrate/20190805140353_remove_rendundant_index_from_releases.rb +++ b/db/migrate/20190805140353_remove_rendundant_index_from_releases.rb @@ -12,10 +12,13 @@ class RemoveRendundantIndexFromReleases < ActiveRecord::Migration[5.2] disable_ddl_transaction! def up - remove_concurrent_index :releases, :project_id + remove_concurrent_index_by_name :releases, 'index_releases_on_project_id' + + # This is an extra index that is not present in db/schema.rb but known to exist on some installs + remove_concurrent_index_by_name :releases, 'releases_project_id_idx' end def down - add_concurrent_index :releases, :project_id + add_concurrent_index :releases, :project_id, name: 'index_releases_on_project_id' end end diff --git a/db/post_migrate/20190911251732_sync_issuables_state_id.rb b/db/post_migrate/20190911251732_sync_issuables_state_id.rb index 4865e98a75e..031837122fb 100644 --- a/db/post_migrate/20190911251732_sync_issuables_state_id.rb +++ b/db/post_migrate/20190911251732_sync_issuables_state_id.rb @@ -62,6 +62,7 @@ class SyncIssuablesStateId < ActiveRecord::Migration[5.2] CASE state WHEN 'opened' THEN 1 WHEN 'closed' THEN 2 + ELSE 2 END SQL end @@ -73,6 +74,7 @@ class SyncIssuablesStateId < ActiveRecord::Migration[5.2] WHEN 'closed' THEN 2 WHEN 'merged' THEN 3 WHEN 'locked' THEN 4 + ELSE 2 END SQL end diff --git a/db/post_migrate/20190926180443_schedule_epic_issues_after_epics_move.rb b/db/post_migrate/20190926180443_schedule_epic_issues_after_epics_move.rb index 4e5b05ce363..86fe0f26681 100644 --- a/db/post_migrate/20190926180443_schedule_epic_issues_after_epics_move.rb +++ b/db/post_migrate/20190926180443_schedule_epic_issues_after_epics_move.rb @@ -24,7 +24,7 @@ class ScheduleEpicIssuesAfterEpicsMove < ActiveRecord::Migration[5.2] Epic.each_batch(of: BATCH_SIZE) do |batch, index| range = batch.pluck('MIN(id)', 'MAX(id)').first - delay = index * interval + delay = index * INTERVAL BackgroundMigrationWorker.perform_in(delay, MIGRATION, *range) end end diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a4fd6bda64e..3f05b80ab63 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -66,6 +66,14 @@ The API can be explored interactively using the [GraphiQL IDE](../index.md#graph | `errors` | String! => Array | Reasons why the mutation failed. | | `note` | Note | The note after mutation | +### CreateEpicPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `epic` | Epic | The created epic | + ### CreateImageDiffNotePayload | Name | Type | Description | diff --git a/doc/api/scim.md b/doc/api/scim.md index 8cbd6103e88..134761334b6 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -5,8 +5,7 @@ The SCIM API implements the [the RFC7644 protocol](https://tools.ietf.org/html/rfc7644). NOTE: **Note:** -[Group SSO](../user/group/saml_sso/index.md) and the feature -flag `:group_scim` must be enabled for the group. For more information, see [SCIM setup documentation](../user/group/saml_sso/scim_setup.md#requirements). +[Group SSO](../user/group/saml_sso/index.md) must be enabled for the group. For more information, see [SCIM setup documentation](../user/group/saml_sso/scim_setup.md#requirements). ## Get a list of SAML users diff --git a/doc/user/analytics/cycle_analytics.md b/doc/user/analytics/cycle_analytics.md index d6e468f3575..c75f101b0e1 100644 --- a/doc/user/analytics/cycle_analytics.md +++ b/doc/user/analytics/cycle_analytics.md @@ -3,9 +3,10 @@ > - Introduced prior to GitLab 12.3 at the project level. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12077) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3 at the group level. -Cycle Analytics measures the time spent to go from an [idea to production] - also known -as cycle time - for each of your projects. Cycle Analytics displays the median time for an idea to -reach production, along with the time typically spent in each DevOps stage along the way. +Cycle Analytics measures the time spent to go from an +[idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) +(also known as cycle time) for each of your projects. Cycle Analytics displays the median time +spent in each stage defined in the process. NOTE: **Note:** Use the `cycle_analytics` feature flag to enable at the group level. @@ -14,8 +15,8 @@ Cycle Analytics is useful in order to quickly determine the velocity of a given project. It points to bottlenecks in the development process, enabling management to uncover, triage, and identify the root cause of slowdowns in the software development life cycle. -Cycle Analytics is tightly coupled with the [GitLab flow] and calculates a separate median for each -stage. +Cycle Analytics is tightly coupled with the [GitLab flow](../../topics/gitlab_flow.md) and +calculates a separate median for each stage. ## Overview @@ -46,6 +47,16 @@ There are seven stages that are tracked as part of the Cycle Analytics calculati - **Production** (Total) - Total lifecycle time; i.e. the velocity of the project or team +## Date ranges + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13216) in GitLab 12.4. + +GitLab provides the ability to filter analytics based on a date range. To filter results: + +1. Select a group. +1. Optionally select a project. +1. Select a date range using the available date pickers. + ## How the data is measured Cycle Analytics records cycle time and data based on the project issues with the @@ -53,7 +64,8 @@ exception of the staging and production stages, where only data deployed to production are measured. Specifically, if your CI is not set up and you have not defined a `production` -or `production/*` [environment], then you will not have any data for those stages. +or `production/*` [environment](../../ci/yaml/README.md#environment), then you will not have any +data for those stages. Each stage of Cycle Analytics is further described in the table below. @@ -64,11 +76,9 @@ Each stage of Cycle Analytics is further described in the table below. | Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request (MR) related to that commit. The key to keep the process tracked is to include the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) to the description of the merge request (for example, `Closes #xxx`, where `xxx` is the number of the issue related to this merge request). If the issue closing pattern is not present in the merge request description, the MR is not considered to the measurement time of the stage. | | Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | | Review | Measures the median time taken to review the merge request that has closing issue pattern, between its creation and until it's merged. | -| Staging | Measures the median time between merging the merge request with closing issue pattern until the very first deployment to production. It's tracked by the [environment] set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI configuration. If there isn't a production environment, this is not tracked. | +| Staging | Measures the median time between merging the merge request with closing issue pattern until the very first deployment to production. It's tracked by the environment set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI configuration. If there isn't a production environment, this is not tracked. | | Production| The sum of all time (medians) taken to run the entire process, from issue creation to deploying the code to production. | ---- - How this works, behind the scenes: 1. Issues and merge requests are grouped together in pairs, such that for each @@ -81,12 +91,12 @@ How this works, behind the scenes: we need for the stages, like issue creation date, merge request merge time, etc. -To sum up, anything that doesn't follow [GitLab flow] will not be tracked and the +To sum up, anything that doesn't follow [GitLab flow](../../workflow/gitlab_flow.md) will not be tracked and the Cycle Analytics dashboard will not present any data for: -- merge requests that do not close an issue. -- issues not labeled with a label present in the Issue Board or for issues not assigned a milestone. -- staging and production stages, if the project has no `production` or `production/*` +- Merge requests that do not close an issue. +- Issues not labeled with a label present in the Issue Board or for issues not assigned a milestone. +- Staging and production stages, if the project has no `production` or `production/*` environment. ## Example workflow @@ -107,7 +117,7 @@ environments is configured. 1. Push branch and create a merge request that contains the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) in its description at 14:00 (stop of **Code** stage / start of **Test** and **Review** stages). -1. The CI starts running your scripts defined in [`.gitlab-ci.yml`][yml] and +1. The CI starts running your scripts defined in [`.gitlab-ci.yml`](../../ci/yaml/README.md) and takes 5min (stop of **Test** stage). 1. Review merge request, ensure that everything is OK and merge the merge request at 19:00. (stop of **Review** stage / start of **Staging** stage). @@ -151,7 +161,7 @@ The current permissions on the Project Cycle Analytics dashboard are: - Internal projects - any authenticated user can access. - Private projects - any member Guest and above can access. -You can [read more about permissions][permissions] in general. +You can [read more about permissions](../../ci/yaml/README.md) in general. NOTE: **Note:** As of GitLab 12.3, the project-level page is deprecated. You should access @@ -169,14 +179,6 @@ For Cycle Analytics functionality introduced in GitLab 12.3 and later: Learn more about Cycle Analytics in the following resources: -- [Cycle Analytics feature page](https://about.gitlab.com/product/cycle-analytics/) -- [Cycle Analytics feature preview](https://about.gitlab.com/blog/2016/09/16/feature-preview-introducing-cycle-analytics/) -- [Cycle Analytics feature highlight](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/) - -[ce-5986]: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/5986 -[ce-20975]: https://gitlab.com/gitlab-org/gitlab-foss/issues/20975 -[environment]: ../../ci/yaml/README.md#environment -[GitLab flow]: ../../topics/gitlab_flow.md -[idea to production]: https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab -[permissions]: ../permissions.md -[yml]: ../../ci/yaml/README.md +- [Cycle Analytics feature page](https://about.gitlab.com/product/cycle-analytics/). +- [Cycle Analytics feature preview](https://about.gitlab.com/blog/2016/09/16/feature-preview-introducing-cycle-analytics/). +- [Cycle Analytics feature highlight](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/). diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index aecbac15c98..40295e47e89 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -42,10 +42,19 @@ The following metrics and visualizations are available on a project or group lev - Number of lines of code per commit. - Number of files touched. - Scatterplot showing all MRs merged on a certain date, together with the days it took to complete the action and a 30 day rolling median. - - Users can zoom in and out on specific days of interest. - Table showing the list of merge requests with their respective time duration metrics. - Users can sort by any of the above metrics. +## Date ranges + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13188) in GitLab 12.4. + +GitLab has the ability to filter analytics based on a date range. To filter results: + +1. Select a group. +1. Optionally select a project. +1. Select a date range using the available date pickers. + ## Permissions The **Productivity Analytics** dashboard can be accessed only: diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index dcb75a19b2a..773f6aaa8a2 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -352,7 +352,10 @@ bottom of the screen with two buttons: Clicking **Submit review** will publish all comments. Any quick actions submitted are performed at this time. -Alternatively, every pending comment has a button to finish the entire review. +Alternatively, to finish the entire review from a pending comment: + +- Click the **Finish review** button on the comment. +- Use the `/submit_review` [quick action](../project/quick_actions.md) in the text of the comment. ![Review submission](img/review_preview.png) diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index 60b779b3f70..7e2b9f7513b 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -25,25 +25,6 @@ The following identity providers are supported: ## Requirements - [Group SSO](index.md) needs to be configured. -- The `scim_group` feature flag must be enabled: - - Run the following commands in a Rails console: - - ```sh - # Omnibus GitLab - gitlab-rails console - - # Installation from source - cd /home/git/gitlab - sudo -u git -H bin/rails console RAILS_ENV=production - ``` - - To enable SCIM for a group named `group_name`: - - ```ruby - group = Group.find_by_full_path('group_name') - Feature.enable(:group_scim, group) - ``` ## GitLab configuration diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 61bc66a6a69..9a9e54d7caa 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -68,7 +68,8 @@ The following quick actions are applicable to descriptions, discussions and thre | `/remove_zoom` | ✓ | | | Remove Zoom meeting from this issue. ([Introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/merge_requests/16609)) | | `/target_branch ` | | ✓ | | Set target branch | | `/wip` | | ✓ | | Toggle the Work In Progress status | -| `/approve` | | ✓ | | Approve the merge request | +| `/approve` | | ✓ | | Approve the merge request **(STARTER)** | +| `/submit_review` | | ✓ | | Submit a pending review. ([Introduced in GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/8041)) **(PREMIUM)** | | `/merge` | | ✓ | | Merge (when pipeline succeeds) | | `/child_epic ` | | | ✓ | Add child epic to ``. The `` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/issues/7330)) **(ULTIMATE)** | | `/remove_child_epic ` | | | ✓ | Remove child epic from ``. The `` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/issues/7330)) **(ULTIMATE)** | diff --git a/jest.config.js b/jest.config.js index c2a512e8afa..f5e589934e6 100644 --- a/jest.config.js +++ b/jest.config.js @@ -28,25 +28,32 @@ if (isESLint) { testMatch = testMatch.map(path => path.replace('_spec.js', '')); } +const moduleNameMapper = { + '^~(/.*)$': '/app/assets/javascripts$1', + '^ee_component(/.*)$': + '/app/assets/javascripts/vue_shared/components/empty_component.js', + '^ee_else_ce(/.*)$': '/app/assets/javascripts$1', + '^helpers(/.*)$': '/spec/frontend/helpers$1', + '^vendor(/.*)$': '/vendor/assets/javascripts$1', + '\\.(jpg|jpeg|png|svg)$': '/spec/frontend/__mocks__/file_mock.js', + 'emojis(/.*).json': '/fixtures/emojis$1.json', + '^spec/test_constants$': '/spec/frontend/helpers/test_constants', +}; + +if (IS_EE) { + const rootDirEE = '/ee/app/assets/javascripts$1'; + Object.assign(moduleNameMapper, { + '^ee(/.*)$': rootDirEE, + '^ee_component(/.*)$': rootDirEE, + '^ee_else_ce(/.*)$': rootDirEE, + }); +} + // eslint-disable-next-line import/no-commonjs module.exports = { testMatch, moduleFileExtensions: ['js', 'json', 'vue'], - moduleNameMapper: { - '^~(/.*)$': '/app/assets/javascripts$1', - '^ee(/.*)$': '/ee/app/assets/javascripts$1', - '^ee_component(/.*)$': IS_EE - ? '/ee/app/assets/javascripts$1' - : '/app/assets/javascripts/vue_shared/components/empty_component.js', - '^ee_else_ce(/.*)$': IS_EE - ? '/ee/app/assets/javascripts$1' - : '/app/assets/javascripts$1', - '^helpers(/.*)$': '/spec/frontend/helpers$1', - '^vendor(/.*)$': '/vendor/assets/javascripts$1', - '\\.(jpg|jpeg|png|svg)$': '/spec/frontend/__mocks__/file_mock.js', - 'emojis(/.*).json': '/fixtures/emojis$1.json', - '^spec/test_constants$': '/spec/frontend/helpers/test_constants', - }, + moduleNameMapper, collectCoverageFrom: ['/app/assets/javascripts/**/*.{js,vue}'], coverageDirectory: '/coverage-frontend/', coverageReporters: ['json', 'lcov', 'text-summary', 'clover'], diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 961012c2cee..910d93f54ce 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -16,7 +16,9 @@ module Gitlab stale_schedule: 'stale schedule', job_execution_timeout: 'job execution timeout', archived_failure: 'archived failure', - unmet_prerequisites: 'unmet prerequisites' + unmet_prerequisites: 'unmet prerequisites', + scheduler_failure: 'scheduler failure', + data_integrity_failure: 'data integrity failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index b2dc92ce010..6d216217bdf 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -58,11 +58,16 @@ module Gitlab # wrong answer. We handle that by querying the full list - which fills # the cache - and using it directly to answer the question. define_method("#{name}_include?") do |value| - if strong_memoized?(name) || !redis_set_cache.exist?(name) - return __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend - end + return __send__(name).include?(value) if strong_memoized?(name) # rubocop:disable GitlabSecurity/PublicSend - redis_set_cache.include?(name, value) + # If the member exists in the set, return as such early. + return true if redis_set_cache.include?(name, value) + + # If it did not, make sure the collection exists. + # If the collection exists, then item does not. + return false if redis_set_cache.exist?(name) + + __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 6d3ac53a787..8035946090a 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -25,7 +25,7 @@ module Gitlab end def read(key) - with { |redis| redis.smembers(cache_key(key)) } + with { |redis| redis.sscan_each(cache_key(key)).to_a } end def write(key, value) @@ -47,11 +47,10 @@ module Gitlab end def fetch(key, &block) - if exist?(key) - read(key) - else - write(key, yield) - end + result = read(key) + return result unless result.empty? + + write(key, yield) end def include?(key, value) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3ff79c23071..a99b275ba23 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -9814,11 +9814,21 @@ msgid_plural "LicenseCompliance|License Compliance detected %d licenses for the msgstr[0] "" msgstr[1] "" +msgid "LicenseCompliance|License Compliance detected %d license for the source branch only; approval required" +msgid_plural "LicenseCompliance|License Compliance detected %d licenses for the source branch only; approval required" +msgstr[0] "" +msgstr[1] "" + msgid "LicenseCompliance|License Compliance detected %d new license" msgid_plural "LicenseCompliance|License Compliance detected %d new licenses" msgstr[0] "" msgstr[1] "" +msgid "LicenseCompliance|License Compliance detected %d new license; approval required" +msgid_plural "LicenseCompliance|License Compliance detected %d new licenses; approval required" +msgstr[0] "" +msgstr[1] "" + msgid "LicenseCompliance|License Compliance detected no licenses for the source branch only" msgstr "" @@ -15967,6 +15977,9 @@ msgstr "" msgid "Subkeys" msgstr "" +msgid "Submit a review" +msgstr "" + msgid "Submit as spam" msgstr "" @@ -15982,6 +15995,12 @@ msgstr "" msgid "Submit search" msgstr "" +msgid "Submit the current review." +msgstr "" + +msgid "Submitted the current review." +msgstr "" + msgid "Subscribe" msgstr "" @@ -18792,6 +18811,9 @@ msgstr "" msgid "Welcome to GitLab" msgstr "" +msgid "Welcome to GitLab %{username}!" +msgstr "" + msgid "Welcome to the Guided GitLab Tour" msgstr "" diff --git a/package.json b/package.json index bf9eb69bdfb..183a978ca75 100644 --- a/package.json +++ b/package.json @@ -37,11 +37,11 @@ "@babel/plugin-syntax-dynamic-import": "^7.2.0", "@babel/plugin-syntax-import-meta": "^7.2.0", "@babel/preset-env": "^7.6.2", - "@gitlab/svgs": "^1.79.0", - "@gitlab/ui": "7.0.0", + "@gitlab/svgs": "^1.80.0", + "@gitlab/ui": "7.3.0", "@gitlab/visual-review-tools": "1.0.3", "apollo-cache-inmemory": "^1.5.1", - "apollo-client": "^2.5.1", + "apollo-client": "^2.6.4", "apollo-link": "^1.2.11", "apollo-link-batch-http": "^1.2.11", "apollo-upload-client": "^10.0.0", diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index ffde883a744..c82075c02f3 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -14,11 +14,11 @@ gitlab: gitaly: resources: requests: - cpu: 600m - memory: 200M - limits: cpu: 1200m - memory: 420M + memory: 240M + limits: + cpu: 1800m + memory: 360M persistence: size: 10G gitlab-exporter: @@ -35,22 +35,22 @@ gitlab: gitlab-shell: resources: requests: - cpu: 125m - memory: 20M + cpu: 230m + memory: 25M limits: - cpu: 250m - memory: 40M + cpu: 345m + memory: 37.5M maxReplicas: 3 hpa: targetAverageValue: 130m sidekiq: resources: requests: - cpu: 500m - memory: 800M + cpu: 650m + memory: 880M limits: - cpu: 1000m - memory: 1.6G + cpu: 975m + memory: 1320M task-runner: resources: requests: @@ -62,11 +62,11 @@ gitlab: unicorn: resources: requests: - cpu: 400m - memory: 1.4G + cpu: 500m + memory: 1540M limits: - cpu: 800m - memory: 2.8G + cpu: 750m + memory: 2310M deployment: readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -75,11 +75,11 @@ gitlab: workhorse: resources: requests: - cpu: 300m - memory: 100M + cpu: 250m + memory: 50M limits: - cpu: 600m - memory: 200M + cpu: 375m + memory: 75M readinessProbe: initialDelaySeconds: 5 # Default is 0 periodSeconds: 15 # Default is 10 @@ -87,11 +87,11 @@ gitlab: gitlab-runner: resources: requests: - cpu: 355m - memory: 300M + cpu: 450m + memory: 100M limits: - cpu: 710m - memory: 600M + cpu: 675m + memory: 150M minio: resources: requests: @@ -108,10 +108,10 @@ nginx-ingress: resources: requests: cpu: 100m - memory: 250M + memory: 450M limits: cpu: 200m - memory: 500M + memory: 675M minAvailable: 1 service: enableHttp: false @@ -133,10 +133,11 @@ postgresql: enabled: false resources: requests: - cpu: 250m - memory: 256M + cpu: 300m + memory: 250M limits: - cpu: 500m + cpu: 450m + memory: 375M prometheus: install: false redis: @@ -157,8 +158,8 @@ registry: minReplicas: 1 resources: requests: - cpu: 50m - memory: 32M - limits: cpu: 100m - memory: 64M + memory: 30M + limits: + cpu: 200m + memory: 45M diff --git a/spec/controllers/projects/releases_controller_spec.rb b/spec/controllers/projects/releases_controller_spec.rb index 63ad633e2fc..357a365e9f1 100644 --- a/spec/controllers/projects/releases_controller_spec.rb +++ b/spec/controllers/projects/releases_controller_spec.rb @@ -56,8 +56,8 @@ describe Projects::ReleasesController do context 'when the project is private and the user is not logged in' do let(:project) { private_project } - it 'renders a 302' do - expect(response.status).to eq(302) + it 'returns a redirect' do + expect(response).to have_gitlab_http_status(:redirect) end end end @@ -78,8 +78,8 @@ describe Projects::ReleasesController do context 'when the project is private and the user is not logged in' do let(:project) { private_project } - it 'renders a 401' do - expect(response.status).to eq(401) + it 'returns a redirect' do + expect(response).to have_gitlab_http_status(:redirect) end end end diff --git a/spec/frontend/vue_shared/components/issue/related_issuable_item_spec.js b/spec/frontend/vue_shared/components/issue/related_issuable_item_spec.js index 575d35c50a9..b85e2673624 100644 --- a/spec/frontend/vue_shared/components/issue/related_issuable_item_spec.js +++ b/spec/frontend/vue_shared/components/issue/related_issuable_item_spec.js @@ -1,7 +1,6 @@ import Vue from 'vue'; import { formatDate } from '~/lib/utils/datetime_utility'; import { mount, createLocalVue } from '@vue/test-utils'; -import { PathIdSeparator } from 'ee/related_issues/constants'; import RelatedIssuableItem from '~/vue_shared/components/issue/related_issuable_item.vue'; import { defaultAssignees, @@ -13,7 +12,7 @@ describe('RelatedIssuableItem', () => { const props = { idKey: 1, displayReference: 'gitlab-org/gitlab-test#1', - pathIdSeparator: PathIdSeparator.Issue, + pathIdSeparator: '#', path: `${gl.TEST_HOST}/path`, title: 'title', confidential: true, diff --git a/spec/migrations/sync_issuables_state_id_spec.rb b/spec/migrations/sync_issuables_state_id_spec.rb index 8d1f8a36ac3..c6e4b504bbb 100644 --- a/spec/migrations/sync_issuables_state_id_spec.rb +++ b/spec/migrations/sync_issuables_state_id_spec.rb @@ -19,19 +19,23 @@ describe SyncIssuablesStateId, :migration, :sidekiq do it 'migrates state column to state_id as integer' do opened_issue = issues.create!(description: 'first', state: 'opened') closed_issue = issues.create!(description: 'second', state: 'closed') + unknown_state_issue = issues.create!(description: 'second', state: 'unknown') opened_merge_request = merge_requests.create!(state: 'opened', target_project_id: project.id, target_branch: 'feature1', source_branch: 'master') closed_merge_request = merge_requests.create!(state: 'closed', target_project_id: project.id, target_branch: 'feature2', source_branch: 'master') merged_merge_request = merge_requests.create!(state: 'merged', target_project_id: project.id, target_branch: 'feature3', source_branch: 'master') locked_merge_request = merge_requests.create!(state: 'locked', target_project_id: project.id, target_branch: 'feature4', source_branch: 'master') + unknown_state_merge_request = merge_requests.create!(state: 'unknown', target_project_id: project.id, target_branch: 'feature4', source_branch: 'master') migrate! expect(opened_issue.reload.state_id).to eq(state_ids[:opened]) expect(closed_issue.reload.state_id).to eq(state_ids[:closed]) + expect(unknown_state_issue.reload.state_id).to eq(state_ids[:closed]) expect(opened_merge_request.reload.state_id).to eq(state_ids[:opened]) expect(closed_merge_request.reload.state_id).to eq(state_ids[:closed]) expect(merged_merge_request.reload.state_id).to eq(state_ids[:merged]) expect(locked_merge_request.reload.state_id).to eq(state_ids[:locked]) + expect(unknown_state_merge_request.reload.state_id).to eq(state_ids[:closed]) end end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 50b104c4095..04c8fb8c5f7 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1587,7 +1587,7 @@ describe Ci::Build do end end - describe '#retries_max' do + describe '#options_retry_max' do context 'with retries max config option' do subject { create(:ci_build, options: { retry: { max: 1 } }) } @@ -1597,7 +1597,7 @@ describe Ci::Build do end it 'returns the number of configured max retries' do - expect(subject.retries_max).to eq 1 + expect(subject.options_retry_max).to eq 1 end end @@ -1607,7 +1607,7 @@ describe Ci::Build do end it 'returns the number of configured max retries' do - expect(subject.retries_max).to eq 1 + expect(subject.options_retry_max).to eq 1 end end end @@ -1615,16 +1615,16 @@ describe Ci::Build do context 'without retries max config option' do subject { create(:ci_build) } - it 'returns zero' do - expect(subject.retries_max).to eq 0 + it 'returns nil' do + expect(subject.options_retry_max).to be_nil end end context 'when build is degenerated' do subject { create(:ci_build, :degenerated) } - it 'returns zero' do - expect(subject.retries_max).to eq 0 + it 'returns nil' do + expect(subject.options_retry_max).to be_nil end end @@ -1632,17 +1632,17 @@ describe Ci::Build do subject { create(:ci_build, options: { retry: 1 }) } it 'returns the number of configured max retries' do - expect(subject.retries_max).to eq 1 + expect(subject.options_retry_max).to eq 1 end end end - describe '#retry_when' do + describe '#options_retry_when' do context 'with retries when config option' do subject { create(:ci_build, options: { retry: { when: ['some_reason'] } }) } it 'returns the configured when' do - expect(subject.retry_when).to eq ['some_reason'] + expect(subject.options_retry_when).to eq ['some_reason'] end end @@ -1650,7 +1650,7 @@ describe Ci::Build do subject { create(:ci_build) } it 'returns always array' do - expect(subject.retry_when).to eq ['always'] + expect(subject.options_retry_when).to eq ['always'] end end @@ -1658,72 +1658,38 @@ describe Ci::Build do subject { create(:ci_build, options: { retry: 1 }) } it 'returns always array' do - expect(subject.retry_when).to eq ['always'] + expect(subject.options_retry_when).to eq ['always'] end end end describe '#retry_failure?' do - subject { create(:ci_build) } + using RSpec::Parameterized::TableSyntax - context 'when retries max is zero' do - before do - expect(subject).to receive(:retries_max).at_least(:once).and_return(0) - end + let(:build) { create(:ci_build) } - it 'returns false' do - expect(subject.retry_failure?).to eq false - end + subject { build.retry_failure? } + + where(:description, :retry_count, :options, :failure_reason, :result) do + "retries are disabled" | 0 | { max: 0 } | nil | false + "max equals count" | 2 | { max: 2 } | nil | false + "max is higher than count" | 1 | { max: 2 } | nil | true + "matching failure reason" | 0 | { when: %w[api_failure], max: 2 } | :api_failure | true + "not matching with always" | 0 | { when: %w[always], max: 2 } | :api_failure | true + "not matching reason" | 0 | { when: %w[script_error], max: 2 } | :api_failure | false + "scheduler failure override" | 1 | { when: %w[scheduler_failure], max: 1 } | :scheduler_failure | false + "default for scheduler failure" | 1 | {} | :scheduler_failure | true end - context 'when retries max equals retries count' do + with_them do before do - expect(subject).to receive(:retries_max).at_least(:once).and_return(1) - expect(subject).to receive(:retries_count).at_least(:once).and_return(1) + allow(build).to receive(:retries_count) { retry_count } + + build.options[:retry] = options + build.failure_reason = failure_reason end - it 'returns false' do - expect(subject.retry_failure?).to eq false - end - end - - context 'when retries max is higher than retries count' do - before do - expect(subject).to receive(:retries_max).at_least(:once).and_return(2) - expect(subject).to receive(:retries_count).at_least(:once).and_return(1) - end - - context 'and retry when is always' do - before do - expect(subject).to receive(:retry_when).at_least(:once).and_return(['always']) - end - - it 'returns true' do - expect(subject.retry_failure?).to eq true - end - end - - context 'and retry when includes the failure_reason' do - before do - expect(subject).to receive(:failure_reason).at_least(:once).and_return('some_reason') - expect(subject).to receive(:retry_when).at_least(:once).and_return(['some_reason']) - end - - it 'returns true' do - expect(subject.retry_failure?).to eq true - end - end - - context 'and retry when does not include failure_reason' do - before do - expect(subject).to receive(:failure_reason).at_least(:once).and_return('some_reason') - expect(subject).to receive(:retry_when).at_least(:once).and_return(['some', 'other failure']) - end - - it 'returns false' do - expect(subject.retry_failure?).to eq false - end - end + it { is_expected.to eq(result) } end end end diff --git a/spec/presenters/ci/build_presenter_spec.rb b/spec/presenters/ci/build_presenter_spec.rb index 11f96962075..b6c47f40ceb 100644 --- a/spec/presenters/ci/build_presenter_spec.rb +++ b/spec/presenters/ci/build_presenter_spec.rb @@ -269,7 +269,7 @@ describe Ci::BuildPresenter do let(:build) { create(:ci_build, :failed, :script_failure) } context 'when is a script or missing dependency failure' do - let(:failure_reasons) { %w(script_failure missing_dependency_failure archived_failure) } + let(:failure_reasons) { %w(script_failure missing_dependency_failure archived_failure scheduler_failure data_integrity_failure) } it 'returns false' do failure_reasons.each do |failure_reason| diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 2c63e621074..a2169a015ee 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -773,8 +773,8 @@ describe Ci::CreatePipelineService do it 'correctly creates builds with auto-retry value configured' do expect(pipeline).to be_persisted - expect(rspec_job.retries_max).to eq 2 - expect(rspec_job.retry_when).to eq ['always'] + expect(rspec_job.options_retry_max).to eq 2 + expect(rspec_job.options_retry_when).to eq ['always'] end end @@ -783,8 +783,8 @@ describe Ci::CreatePipelineService do it 'correctly creates builds with auto-retry value configured' do expect(pipeline).to be_persisted - expect(rspec_job.retries_max).to eq 2 - expect(rspec_job.retry_when).to eq ['runner_system_failure'] + expect(rspec_job.options_retry_max).to eq 2 + expect(rspec_job.options_retry_when).to eq ['runner_system_failure'] end end end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 2f2c525ccc4..04334fb8915 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -502,6 +502,57 @@ module Ci end end + context 'when build has data integrity problem' do + let!(:pending_job) do + create(:ci_build, :pending, pipeline: pipeline) + end + + before do + pending_job.update_columns(options: "string") + end + + subject { execute(specific_runner, {}) } + + it 'does drop the build and logs both failures' do + expect(Gitlab::Sentry).to receive(:track_acceptable_exception) + .with(anything, a_hash_including(extra: a_hash_including(build_id: pending_job.id))) + .twice + .and_call_original + + expect(subject).to be_nil + + pending_job.reload + expect(pending_job).to be_failed + expect(pending_job).to be_data_integrity_failure + end + end + + context 'when build fails to be run!' do + let!(:pending_job) do + create(:ci_build, :pending, pipeline: pipeline) + end + + before do + expect_any_instance_of(Ci::Build).to receive(:run!) + .and_raise(RuntimeError, 'scheduler error') + end + + subject { execute(specific_runner, {}) } + + it 'does drop the build and logs failure' do + expect(Gitlab::Sentry).to receive(:track_acceptable_exception) + .with(anything, a_hash_including(extra: a_hash_including(build_id: pending_job.id))) + .once + .and_call_original + + expect(subject).to be_nil + + pending_job.reload + expect(pending_job).to be_failed + expect(pending_job).to be_scheduler_failure + end + end + context 'when an exception is raised during a persistent ref creation' do before do allow_any_instance_of(Ci::PersistentRef).to receive(:exist?) { false } diff --git a/spec/services/concerns/merge_requests/assigns_merge_params_spec.rb b/spec/services/concerns/merge_requests/assigns_merge_params_spec.rb index 5b653aa331c..9cf7f354191 100644 --- a/spec/services/concerns/merge_requests/assigns_merge_params_spec.rb +++ b/spec/services/concerns/merge_requests/assigns_merge_params_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe MergeRequests::AssignsMergeParams do diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore old mode 100755 new mode 100644 diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore old mode 100755 new mode 100644 diff --git a/yarn.lock b/yarn.lock index 54221ffc2ce..9beb499025c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -990,15 +990,15 @@ dependencies: vue-eslint-parser "^6.0.4" -"@gitlab/svgs@^1.79.0": - version "1.79.0" - resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.79.0.tgz#7e79666118d6adc0247bdb3b3b6b2b299aa5a439" - integrity sha512-0pTUviQqwyaKBOB6OL7Mmr2dQn/dGB03XslBMtL9lFZz1baB7d6xf+zxFU0GBAJUJan397IbBddE1jjUAQT8Fw== +"@gitlab/svgs@^1.80.0": + version "1.80.0" + resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.80.0.tgz#52b2d25f002cdfe9bd7c366a043c1849687ad64b" + integrity sha512-hsyX3EZV/hk9bMTvvoxVcNC0EO6sy771BC2vXjqGtzjye4hTs0BTAzu3V0UPWuDompHtKXi/plVcJU+NxNLQ6Q== -"@gitlab/ui@7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-7.0.0.tgz#74ca106856a47e7793df0b7448cbf2903634709c" - integrity sha512-uiRV2VuGpWcCpj+d4M6VA8ItyLkTxfzGnaNbyX8UcswInAKjMMDO7KspF9DWlFDJVH5qzDguOSgons9NQJy53g== +"@gitlab/ui@7.3.0": + version "7.3.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-7.3.0.tgz#9ed6d2144cb999c12675b309ecda3279c4b88bf6" + integrity sha512-QMn84x7DrjDOCKD1/Exh26wwkAvdAjlIWjafvISTHZ+PAWY6XxEAYyjllM5k0fQpNZP3sw7sBWWYvezDVdLnmw== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" @@ -1531,6 +1531,13 @@ "@webassemblyjs/wast-parser" "1.8.5" "@xtuc/long" "4.2.2" +"@wry/equality@^0.1.2": + version "0.1.9" + resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.1.9.tgz#b13e18b7a8053c6858aa6c85b54911fb31e3a909" + integrity sha512-mB6ceGjpMGz1ZTza8HYnrPGos2mC6So4NhS1PtZ8s4Qt0K7fBiIGhpSxUbQmhwcSWE3no+bYxmI2OL6KuXYmoQ== + dependencies: + tslib "^1.9.3" + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -1706,26 +1713,25 @@ apollo-cache-inmemory@^1.5.1: ts-invariant "^0.2.1" tslib "^1.9.3" -apollo-cache@1.2.1, apollo-cache@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/apollo-cache/-/apollo-cache-1.2.1.tgz#aae71eb4a11f1f7322adc343f84b1a39b0693644" - integrity sha512-nzFmep/oKlbzUuDyz6fS6aYhRmfpcHWqNkkA9Bbxwk18RD6LXC4eZkuE0gXRX0IibVBHNjYVK+Szi0Yied4SpQ== +apollo-cache@1.3.2, apollo-cache@^1.2.1: + version "1.3.2" + resolved "https://registry.yarnpkg.com/apollo-cache/-/apollo-cache-1.3.2.tgz#df4dce56240d6c95c613510d7e409f7214e6d26a" + integrity sha512-+KA685AV5ETEJfjZuviRTEImGA11uNBp/MJGnaCvkgr+BYRrGLruVKBv6WvyFod27WEB2sp7SsG8cNBKANhGLg== dependencies: - apollo-utilities "^1.2.1" + apollo-utilities "^1.3.2" tslib "^1.9.3" -apollo-client@^2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/apollo-client/-/apollo-client-2.5.1.tgz#36126ed1d32edd79c3713c6684546a3bea80e6d1" - integrity sha512-MNcQKiqLHdGmNJ0rZ0NXaHrToXapJgS/5kPk0FygXt+/FmDCdzqcujI7OPxEC6e9Yw5S/8dIvOXcRNuOMElHkA== +apollo-client@^2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/apollo-client/-/apollo-client-2.6.4.tgz#872c32927263a0d34655c5ef8a8949fbb20b6140" + integrity sha512-oWOwEOxQ9neHHVZrQhHDbI6bIibp9SHgxaLRVPoGvOFy7OH5XUykZE7hBQAVxq99tQjBzgytaZffQkeWo1B4VQ== dependencies: "@types/zen-observable" "^0.8.0" - apollo-cache "1.2.1" + apollo-cache "1.3.2" apollo-link "^1.0.0" - apollo-link-dedup "^1.0.0" - apollo-utilities "1.2.1" + apollo-utilities "1.3.2" symbol-observable "^1.0.2" - ts-invariant "^0.2.1" + ts-invariant "^0.4.0" tslib "^1.9.3" zen-observable "^0.8.0" @@ -1747,13 +1753,6 @@ apollo-link-batch@^1.1.12: apollo-link "^1.2.11" tslib "^1.9.3" -apollo-link-dedup@^1.0.0: - version "1.0.10" - resolved "https://registry.yarnpkg.com/apollo-link-dedup/-/apollo-link-dedup-1.0.10.tgz#7b94589fe7f969777efd18a129043c78430800ae" - integrity sha512-tpUI9lMZsidxdNygSY1FxflXEkUZnvKRkMUsXXuQUNoSLeNtEvUX7QtKRAl4k9ubLl8JKKc9X3L3onAFeGTK8w== - dependencies: - apollo-link "^1.2.3" - apollo-link-http-common@^0.2.13, apollo-link-http-common@^0.2.8: version "0.2.13" resolved "https://registry.yarnpkg.com/apollo-link-http-common/-/apollo-link-http-common-0.2.13.tgz#c688f6baaffdc7b269b2db7ae89dae7c58b5b350" @@ -1763,7 +1762,7 @@ apollo-link-http-common@^0.2.13, apollo-link-http-common@^0.2.8: ts-invariant "^0.3.2" tslib "^1.9.3" -apollo-link@^1.0.0, apollo-link@^1.2.11, apollo-link@^1.2.3, apollo-link@^1.2.6: +apollo-link@^1.0.0, apollo-link@^1.2.11, apollo-link@^1.2.6: version "1.2.11" resolved "https://registry.yarnpkg.com/apollo-link/-/apollo-link-1.2.11.tgz#493293b747ad3237114ccd22e9f559e5e24a194d" integrity sha512-PQvRCg13VduLy3X/0L79M6uOpTh5iHdxnxYuo8yL7sJlWybKRJwsv4IcRBJpMFbChOOaHY7Og9wgPo6DLKDKDA== @@ -1782,13 +1781,14 @@ apollo-upload-client@^10.0.0: apollo-link-http-common "^0.2.8" extract-files "^5.0.0" -apollo-utilities@1.2.1, apollo-utilities@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/apollo-utilities/-/apollo-utilities-1.2.1.tgz#1c3a1ebf5607d7c8efe7636daaf58e7463b41b3c" - integrity sha512-Zv8Udp9XTSFiN8oyXOjf6PMHepD4yxxReLsl6dPUy5Ths7jti3nmlBzZUOxuTWRwZn0MoclqL7RQ5UEJN8MAxg== +apollo-utilities@1.3.2, apollo-utilities@^1.2.1, apollo-utilities@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/apollo-utilities/-/apollo-utilities-1.3.2.tgz#8cbdcf8b012f664cd6cb5767f6130f5aed9115c9" + integrity sha512-JWNHj8XChz7S4OZghV6yc9FNnzEXj285QYp/nLNh943iObycI5GTDO3NGR9Dth12LRrSFMeDOConPfPln+WGfg== dependencies: + "@wry/equality" "^0.1.2" fast-json-stable-stringify "^2.0.0" - ts-invariant "^0.2.1" + ts-invariant "^0.4.0" tslib "^1.9.3" append-buffer@^1.0.2: @@ -12051,6 +12051,13 @@ ts-invariant@^0.3.2: dependencies: tslib "^1.9.3" +ts-invariant@^0.4.0: + version "0.4.4" + resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.4.4.tgz#97a523518688f93aafad01b0e80eb803eb2abd86" + integrity sha512-uEtWkFM/sdZvRNNDL3Ehu4WVpwaulhwQszV8mrtcdeE8nN00BV9mAmQ88RkrBhFgl9gMgvjJLAQcZbnPXI9mlA== + dependencies: + tslib "^1.9.3" + ts-jest@24.0.0, ts-jest@^23.10.5: version "24.0.0" resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-24.0.0.tgz#3f26bf2ec1fa584863a5a9c29bd8717d549efbf6"