Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-03-18 03:07:42 +00:00
parent 8a55899cab
commit 5ea8a46ef4
14 changed files with 157 additions and 211 deletions

View file

@ -91,6 +91,8 @@ variables:
CHECK_PRECOMPILED_ASSETS: "true"
FF_USE_FASTZIP: "true"
SKIP_FLAKY_TESTS_AUTOMATICALLY: "true"
# Run with decomposed databases by default
DECOMPOSED_DB: "true"
DOCS_REVIEW_APPS_DOMAIN: "178.62.207.141.nip.io"
DOCS_GITLAB_REPO_SUFFIX: "ee"

View file

@ -18,12 +18,14 @@
variables:
RSPEC_TESTS_MAPPING_ENABLED: "true"
.decomposed-database:
.single-db:
variables:
DECOMPOSED_DB: "true"
DECOMPOSED_DB: "false"
.decomposed-database-rspec:
extends: .decomposed-database
.single-db-rspec:
extends: .single-db
variables:
GITLAB_USE_MODEL_LOAD_BALANCING: "false"
.rspec-base:
extends:
@ -274,11 +276,11 @@ rspec migration pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-migration:minimal
rspec migration pg12 decomposed:
rspec migration pg12 single-db:
extends:
- rspec migration pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec unit pg12:
extends:
@ -292,11 +294,11 @@ rspec unit pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-unit:minimal
rspec unit pg12 decomposed:
rspec unit pg12 single-db:
extends:
- rspec unit pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec integration pg12:
extends:
@ -310,11 +312,11 @@ rspec integration pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-integration:minimal
rspec integration pg12 decomposed:
rspec integration pg12 single-db:
extends:
- rspec integration pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec system pg12:
extends:
@ -330,11 +332,11 @@ rspec system pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-and-foss-system:minimal
rspec system pg12 decomposed:
rspec system pg12 single-db:
extends:
- rspec system pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
# Dedicated job to test DB library code against PG11.
# Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs.
@ -361,22 +363,25 @@ db:rollback:
- scripts/db_tasks db:migrate VERSION=20210301200959
- scripts/db_tasks db:migrate SKIP_SCHEMA_VERSION_CHECK=true
db:rollback decomposed:
db:rollback single-db:
extends:
- db:rollback
- .decomposed-database
- .rails:rules:decomposed-databases
- .single-db
- .rails:rules:single-db
db:migrate:reset:
extends: .db-job-base
script:
- bundle exec rake db:migrate:reset
variables:
# FIXME: make this job work with `GITLAB_USE_MODEL_LOAD_BALANCING: true`, see https://gitlab.com/gitlab-org/gitlab/-/issues/355573
GITLAB_USE_MODEL_LOAD_BALANCING: "false"
db:migrate:reset decomposed:
db:migrate:reset single-db:
extends:
- db:migrate:reset
- .decomposed-database
- .rails:rules:decomposed-databases
- .single-db
- .rails:rules:single-db
db:migrate-from-previous-major-version:
extends: .db-job-base
@ -385,6 +390,7 @@ db:migrate-from-previous-major-version:
SETUP_DB: "false"
PROJECT_TO_CHECKOUT: "gitlab-foss"
TAG_TO_CHECKOUT: "v13.12.9"
# FIXME: make this job work with `GITLAB_USE_MODEL_LOAD_BALANCING: true`, see https://gitlab.com/gitlab-org/gitlab/-/issues/355573
GITLAB_USE_MODEL_LOAD_BALANCING: "false"
before_script:
- !reference [.default-before_script, before_script]
@ -399,11 +405,11 @@ db:migrate-from-previous-major-version:
script:
- run_timed_command "scripts/db_tasks db:migrate"
db:migrate-from-previous-major-version-decomposed:
db:migrate-from-previous-major-version-single-db:
extends:
- db:migrate-from-previous-major-version
- .decomposed-database
- .rails:rules:decomposed-databases
- .single-db
- .rails:rules:single-db
.db:check-schema-base:
extends:
@ -420,9 +426,9 @@ db:check-schema:
- db:migrate-from-previous-major-version
- .db:check-schema-base
db:check-schema-decomposed:
db:check-schema-single-db:
extends:
- db:migrate-from-previous-major-version-decomposed
- db:migrate-from-previous-major-version-single-db
- .db:check-schema-base
db:check-migrations:
@ -434,11 +440,11 @@ db:check-migrations:
- scripts/validate_migration_schema
allow_failure: true
db:check-migrations-decomposed:
db:check-migrations-single-db:
extends:
- db:check-migrations
- .decomposed-database
- .rails:rules:decomposed-databases
- .single-db
- .rails:rules:single-db
db:migrate-non-superuser:
extends:
@ -545,11 +551,11 @@ rspec:coverage:
- rspec unit pg12 minimal
- rspec integration pg12 minimal
- rspec system pg12 minimal
# FOSS/EE decomposed jobs
- rspec migration pg12 decomposed
- rspec unit pg12 decomposed
- rspec integration pg12 decomposed
- rspec system pg12 decomposed
# FOSS/EE single-db jobs
- rspec migration pg12 single-db
- rspec unit pg12 single-db
- rspec integration pg12 single-db
- rspec system pg12 single-db
# EE jobs
- rspec-ee migration pg12
- rspec-ee unit pg12
@ -560,11 +566,11 @@ rspec:coverage:
- rspec-ee unit pg12 minimal
- rspec-ee integration pg12 minimal
- rspec-ee system pg12 minimal
# EE decomposed jobs
- rspec-ee migration pg12 decomposed
- rspec-ee unit pg12 decomposed
- rspec-ee integration pg12 decomposed
- rspec-ee system pg12 decomposed
# EE single-db jobs
- rspec-ee migration pg12 single-db
- rspec-ee unit pg12 single-db
- rspec-ee integration pg12 single-db
- rspec-ee system pg12 single-db
# Geo jobs
- rspec-ee unit pg12 geo
- rspec-ee integration pg12 geo
@ -578,16 +584,16 @@ rspec:coverage:
# As-if-FOSS jobs
- rspec migration pg12-as-if-foss
- rspec migration pg12-as-if-foss minimal
- rspec migration pg12-as-if-foss decomposed
- rspec migration pg12-as-if-foss single-db
- rspec unit pg12-as-if-foss
- rspec unit pg12-as-if-foss minimal
- rspec unit pg12-as-if-foss decomposed
- rspec unit pg12-as-if-foss single-db
- rspec integration pg12-as-if-foss
- rspec integration pg12-as-if-foss minimal
- rspec integration pg12-as-if-foss decomposed
- rspec integration pg12-as-if-foss single-db
- rspec system pg12-as-if-foss
- rspec system pg12-as-if-foss minimal
- rspec system pg12-as-if-foss decomposed
- rspec system pg12-as-if-foss single-db
script:
- run_timed_command "bundle exec scripts/merge-simplecov"
coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
@ -677,11 +683,11 @@ rspec migration pg12-as-if-foss minimal:
- .minimal-rspec-tests
- .rails:rules:as-if-foss-migration:minimal
rspec migration pg12-as-if-foss decomposed:
rspec migration pg12-as-if-foss single-db:
extends:
- rspec migration pg12-as-if-foss
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec unit pg12-as-if-foss:
extends:
@ -695,11 +701,11 @@ rspec unit pg12-as-if-foss minimal:
- .minimal-rspec-tests
- .rails:rules:as-if-foss-unit:minimal
rspec unit pg12-as-if-foss decomposed:
rspec unit pg12-as-if-foss single-db:
extends:
- rspec unit pg12-as-if-foss
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec integration pg12-as-if-foss:
extends:
@ -713,11 +719,11 @@ rspec integration pg12-as-if-foss minimal:
- .minimal-rspec-tests
- .rails:rules:as-if-foss-integration:minimal
rspec integration pg12-as-if-foss decomposed:
rspec integration pg12-as-if-foss single-db:
extends:
- rspec integration pg12-as-if-foss
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec system pg12-as-if-foss:
extends:
@ -731,11 +737,11 @@ rspec system pg12-as-if-foss minimal:
- .minimal-rspec-tests
- .rails:rules:as-if-foss-system:minimal
rspec system pg12-as-if-foss decomposed:
rspec system pg12-as-if-foss single-db:
extends:
- rspec system pg12-as-if-foss
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec migration pg12-as-if-jh:
extends:
@ -775,11 +781,11 @@ rspec-ee migration pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-only-migration:minimal
rspec-ee migration pg12 decomposed:
rspec-ee migration pg12 single-db:
extends:
- rspec-ee migration pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec-ee unit pg12:
extends:
@ -793,11 +799,11 @@ rspec-ee unit pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-only-unit:minimal
rspec-ee unit pg12 decomposed:
rspec-ee unit pg12 single-db:
extends:
- rspec-ee unit pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec-ee integration pg12:
extends:
@ -811,11 +817,11 @@ rspec-ee integration pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-only-integration:minimal
rspec-ee integration pg12 decomposed:
rspec-ee integration pg12 single-db:
extends:
- rspec-ee integration pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec-ee system pg12:
extends:
@ -829,11 +835,11 @@ rspec-ee system pg12 minimal:
- .minimal-rspec-tests
- .rails:rules:ee-only-system:minimal
rspec-ee system pg12 decomposed:
rspec-ee system pg12 single-db:
extends:
- rspec-ee system pg12
- .decomposed-database-rspec
- .rails:rules:decomposed-databases
- .single-db-rspec
- .rails:rules:single-db
rspec-ee unit pg12 geo:
extends:

View file

@ -61,8 +61,8 @@
.if-merge-request-labels-run-all-jest: &if-merge-request-labels-run-all-jest
if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-all-jest/'
.if-merge-request-labels-run-decomposed: &if-merge-request-labels-run-decomposed
if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-decomposed/'
.if-merge-request-labels-run-single-db: &if-merge-request-labels-run-single-db
if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-single-db/'
.if-merge-request-labels-run-review-app: &if-merge-request-labels-run-review-app
if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-review-app/'
@ -907,9 +907,11 @@
- changes: *setup-test-env-patterns
- <<: *if-merge-request-labels-run-all-rspec
.rails:rules:decomposed-databases:
.rails:rules:single-db:
rules:
- <<: *if-merge-request-labels-run-decomposed
- <<: *if-merge-request-labels-run-single-db
- <<: *if-merge-request
changes: *db-patterns
- <<: *if-default-branch-schedule-nightly
.rails:rules:ee-and-foss-migration:

View file

@ -1,4 +1,22 @@
#
# PRODUCTION
#
production:
main:
adapter: postgresql
encoding: unicode
database: gitlabhq_production
username: git
password: "secure password"
host: localhost
ci:
adapter: postgresql
encoding: unicode
database: gitlabhq_production_ci
username: git
password: "secure password"
host: localhost
#
# Development specific
#
development:

View file

@ -52,8 +52,8 @@ mutation {
errors
externalAuditEventDestination {
destinationUrl
group {
verificationToken
group {
name
}
}

View file

@ -81,11 +81,10 @@ Maintainer role.
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342180/) in GitLab 14.9
You can use the UI or API to take these actions on a deployment:
Using either the GitLab UI or the API, you can:
- Approve it
- Allow it to proceed
- Reject it
- Approve a deployment to allow it to proceed.
- Reject a deployment to prevent it.
### Approve or reject a deployment using the UI

View file

@ -194,6 +194,14 @@ We keep track of retried tests in the `$RETRIED_TESTS_REPORT_FILE` file saved as
See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148).
### Single database testing
By default, all tests run with [multiple databases](database/multiple_databases.md).
We also run tests with a single database in nightly scheduled pipelines, and in merge requests that touch database-related files.
If you want to force tests to run with a single database, you can add the `pipeline:run-single-db` label to the merge request.
### Monitoring
The GitLab test suite is [monitored](performance.md#rspec-profiling) for the `main` branch, and any branch

View file

@ -14,14 +14,14 @@ module Atlassian
def send_info(project:, update_sequence_id: nil, **args)
common = { project: project, update_sequence_id: update_sequence_id }
dev_info = DevInfo.new(**common.merge(args.slice(:commits, :branches, :merge_requests)))
dev_info = args.slice(:commits, :branches, :merge_requests)
build_info = args.slice(:pipelines)
deploy_info = args.slice(:deployments)
ff_info = args.slice(:feature_flags)
responses = []
responses << store_dev_info(dev_info) if dev_info.present?
responses << store_dev_info(**common, **dev_info) if dev_info.present?
responses << store_build_info(**common, **build_info) if build_info.present?
responses << store_deploy_info(**common, **deploy_info) if deploy_info.present?
responses << store_ff_info(**common, **ff_info) if ff_info.present?
@ -93,8 +93,17 @@ module Atlassian
handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds') }
end
def store_dev_info(dev_info)
post(dev_info.url, dev_info.body)
def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
project,
commits: commits,
branches: branches,
merge_requests: merge_requests,
user_notes_count: user_notes_count(merge_requests),
update_sequence_id: update_sequence_id
)
post('/rest/devinfo/0.10/bulk', { repositories: [repo] })
end
def post(path, payload)
@ -148,6 +157,14 @@ module Atlassian
{ 'errorMessages' => messages }
end
def user_notes_count(merge_requests)
return unless merge_requests
Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').to_h do |count_group|
[count_group.noteable_id, count_group.count]
end
end
def jwt_token(http_method, uri)
claims = Atlassian::Jwt.build_claims(
Atlassian::JiraConnect.app_key,

View file

@ -1,48 +0,0 @@
# frozen_string_literal: true
module Atlassian
module JiraConnect
class DevInfo
URL = '/rest/devinfo/0.10/bulk'
def initialize(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
@project = project
@commits = commits
@branches = branches
@merge_requests = merge_requests
@update_sequence_id = update_sequence_id
end
def url
URL
end
def body
repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
@project,
commits: @commits,
branches: @branches,
merge_requests: @merge_requests,
user_notes_count: user_notes_count,
update_sequence_id: @update_sequence_id
)
{ repositories: [repo] }
end
def present?
[@commits, @branches, @merge_requests].any?(&:present?)
end
private
def user_notes_count
return unless @merge_requests
Note.count_for_collection(@merge_requests.map(&:id), 'MergeRequest').to_h do |count_group|
[count_group.noteable_id, count_group.count]
end
end
end
end
end

View file

@ -67,19 +67,18 @@ end
namespace :test_resources do
desc "Deletes resources created during E2E test runs"
task :delete, :file_pattern do |t, args|
args.with_defaults(file_pattern: QA::Runtime::Env.test_resources_created_filepath)
task :delete, [:file_pattern] do |t, args|
QA::Tools::TestResourcesHandler.new(args[:file_pattern]).run_delete
end
desc "Upload test resources JSON files to GCS"
task :upload, [:file_pattern, :environment_name] do |t, args|
QA::Tools::TestResourcesHandler.new(args[:file_pattern]).upload(args[:environment_name])
task :upload, [:file_pattern, :ci_project_name] do |t, args|
QA::Tools::TestResourcesHandler.new(args[:file_pattern]).upload(args[:ci_project_name])
end
desc "Download test resources JSON files from GCS"
task :download, [:environment_name] do |t, args|
QA::Tools::TestResourcesHandler.new.download(args[:environment_name])
task :download, [:ci_project_name] do |t, args|
QA::Tools::TestResourcesHandler.new.download(args[:ci_project_name])
end
end
# rubocop:enable Rails/RakeEnvironment

View file

@ -5,16 +5,15 @@ require "fog/google"
# This script handles resources created during E2E test runs
#
# Delete: find all matching file pattern, read file and delete resources
# rake test_resources:delete
# rake test_resources:delete[<file_pattern>]
#
# Upload: find all matching file pattern for failed test resources
# upload these files to GCS bucket `failed-test-resources` under specific environment name
# rake test_resources:upload[<file_pattern>,<environment_name>]
# rake test_resources:upload[<file_pattern>,<ci_project_name>]
#
# Download: download JSON files under a given environment name (bucket directory)
# save to local under `tmp/`
# rake test_resources:download[<environment_name>]
# rake test_resources:download[<ci_project_name>]
#
# Required environment variables:
# GITLAB_ADDRESS, required for delete task
@ -61,11 +60,11 @@ module QA
# Files are organized by environment in which tests were executed
#
# E.g: staging/failed-test-resources-<randomhex>.json
def upload(environment_name)
def upload(ci_project_name)
return puts "\nNothing to upload!" if files.empty?
files.each do |file|
file_name = "#{environment_name}/#{file.split('/').last}"
file_name = "#{ci_project_name}/#{file.split('/').last}"
Runtime::Logger.info("Uploading #{file_name}...")
gcs_storage.put_object(BUCKET, file_name, File.read(file))
end
@ -75,8 +74,8 @@ module QA
# Download files from GCS bucket by environment name
# Delete the files afterward
def download(environment_name)
files_list = gcs_storage.list_objects(BUCKET, prefix: environment_name).items.each_with_object([]) do |obj, arr|
def download(ci_project_name)
files_list = gcs_storage.list_objects(BUCKET, prefix: ci_project_name).items.each_with_object([]) do |obj, arr|
arr << obj.name
end

View file

@ -1,8 +0,0 @@
{
"repositories": {
"type": "array",
"items": {
"$ref": "./repository.json"
}
}
}

View file

@ -58,16 +58,12 @@ RSpec.describe Atlassian::JiraConnect::Client do
deployments: :q
).and_return(:deploys_stored)
expect(Atlassian::JiraConnect::DevInfo).to receive(:new).with(
expect(subject).to receive(:store_dev_info).with(
project: project,
update_sequence_id: :x,
commits: :a,
branches: :b,
merge_requests: :c
).and_call_original
expect(subject).to receive(:store_dev_info).with(
instance_of(Atlassian::JiraConnect::DevInfo)
).and_return(:dev_stored)
args = {
@ -87,7 +83,9 @@ RSpec.describe Atlassian::JiraConnect::Client do
it 'only calls methods that we need to call' do
expect(subject).to receive(:store_dev_info).with(
instance_of(Atlassian::JiraConnect::DevInfo)
project: project,
update_sequence_id: :x,
commits: :a
).and_return(:dev_stored)
args = {
@ -404,7 +402,15 @@ RSpec.describe Atlassian::JiraConnect::Client do
end
it "calls the API with auth headers" do
subject.send(:store_dev_info, Atlassian::JiraConnect::DevInfo.new(project: project))
subject.send(:store_dev_info, project: project)
end
it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.count
merge_requests << create(:merge_request, :unique_branches)
expect { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.not_to exceed_query_limit(control_count)
end
end
end

View file

@ -1,54 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Atlassian::JiraConnect::DevInfo do
let_it_be(:project) { create_default(:project, :repository).freeze }
let(:update_sequence_id) { '123' }
describe '#url' do
subject { described_class.new(project: project).url }
it { is_expected.to eq('/rest/devinfo/0.10/bulk') }
end
describe '#body' do
let_it_be(:merge_request) { create(:merge_request, :unique_branches, title: 'TEST-123') }
let_it_be(:note) { create(:note, noteable: merge_request, project: merge_request.project) }
let_it_be(:branches) do
project.repository.create_branch('TEST-123', project.default_branch_or_main)
[project.repository.find_branch('TEST-123')]
end
let(:merge_requests) { [merge_request] }
subject(:body) { described_class.new(project: project, branches: branches, merge_requests: merge_requests, update_sequence_id: update_sequence_id).body.to_json }
it 'matches the schema' do
expect(body).to match_schema('jira_connect/dev_info')
end
it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new { subject }.count
merge_requests << create(:merge_request, :unique_branches)
expect { subject }.not_to exceed_query_limit(control_count)
end
end
describe '#present?' do
let(:arguments) { { commits: nil, branches: nil, merge_requests: nil } }
subject { described_class.new(**{ project: project, update_sequence_id: update_sequence_id }.merge(arguments)).present? }
it { is_expected.to eq(false) }
context 'with commits, branches or merge requests' do
let(:arguments) { { commits: anything, branches: anything, merge_requests: anything } }
it { is_expected.to eq(true) }
end
end
end