kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-11-24 15:14:19 +00:00
parent 77b8390171
commit b8d516a687
79 changed files with 1588 additions and 557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab/ci/review-apps/main.gitlab-ci.yml
View File

@ -65,6 +65,7 @@ review-deploy:
- deploy || (display_deployment_debug && exit 1)
- verify_deploy || exit 1
- disable_sign_ups || (delete_release && exit 1)
- create_sample_projects
after_script:
# Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan.
# Set DAST_RUN to true when jobs are manually scheduled.

3
.gitlab/merge_request_templates/Documentation.md
View File

@ -20,7 +20,6 @@
If you are only adding documentation, do not add any of the following labels:
- `~"type::feature"`
- `~"frontend"`
- `~"backend"`
- `~"type::bug"`
@ -44,5 +43,5 @@ Documentation-related MRs should be reviewed by a Technical Writer for a non-blo
- [ ] Review by assigned maintainer, who can always request/require the above reviews. Maintainer's review can occur before or after a technical writer review.
- [ ] Ensure a release milestone is set.
/label ~documentation
/label ~documentation ~"type::maintenance"
/assign me

2
GITLAB_SHELL_VERSION
View File

@ -1 +1 @@
13.22.0
13.22.1

4
app/assets/javascripts/notes/components/notes_app.vue
View File

@ -104,6 +104,10 @@ export default {
});
}
if (this.sortDirDesc) {
return skeletonNotes.concat(this.discussions);
}
return this.discussions.concat(skeletonNotes);
},
canReply() {

33
app/assets/stylesheets/framework/files.scss
View File

@ -151,7 +151,17 @@
margin: 0;
}
//
// IMPORTANT PERFORMANCE OPTIMIZATION
//
// When viewinng a blame with many commits a lot of content is rendered on the page.
// content-visibility rule below ensure that we only render what is visible to the user,
// thus reducing TBT in the browser.
// Grid is used instead of table layout because content-visibility performs better with it.
tr {
content-visibility: auto;
display: grid;
grid-template-columns: 400px max-content auto;
border-bottom: 1px solid $gray-darker;
&:last-child {
@ -201,6 +211,10 @@
&.lines {
padding: 0;
}
.code {
height: 100%;
}
}
@for $i from 0 through 5 {
@ -222,25 +236,6 @@
color: $gray-900;
}
}
//
// IMPORTANT PERFORMANCE OPTIMIZATION
//
// When viewinng a blame with many commits a lot of content is rendered on the page.
// content-visibility rules below ensure that we only render what is visible to the user, thus reducing TBT in the browser.
.commit {
content-visibility: auto;
contain-intrinsic-size: 1px 3em;
}
code .line {
content-visibility: auto;
contain-intrinsic-size: 1px 1.1875rem;
}
.line-numbers {
content-visibility: auto;
}
}
&.logs {

8
app/controllers/clusters/clusters_controller.rb
View File

@ -280,7 +280,10 @@ class Clusters::ClustersController < Clusters::BaseController
end
def generate_gcp_authorize_url
state = generate_session_key_redirect(clusterable.new_path(provider: :gcp).to_s)
new_path = clusterable.new_path(provider: :gcp).to_s
error_path = @project ? project_clusters_path(@project) : new_path
state = generate_session_key_redirect(new_path, error_path)
@authorize_url = GoogleApi::CloudPlatform::Client.new(
nil, callback_google_api_auth_url,
@ -339,9 +342,10 @@ class Clusters::ClustersController < Clusters::BaseController
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at]
end
def generate_session_key_redirect(uri)
def generate_session_key_redirect(uri, error_uri)
GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key|
session[key] = uri
session[:error_uri] = error_uri
end
end

33
app/controllers/google_api/authorizations_controller.rb
View File

@ -8,19 +8,36 @@ module GoogleApi
feature_category :kubernetes_management
##
# handle the response from google after the user
# goes through authentication and authorization process
def callback
token, expires_at = GoogleApi::CloudPlatform::Client
.new(nil, callback_google_api_auth_url)
.get_token(params[:code])
session[GoogleApi::CloudPlatform::Client.session_key_for_token] = token
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] =
expires_at.to_s
redirect_uri = redirect_uri_from_session
##
# when the user declines authorizations
# `error` param is returned
if params[:error]
flash[:alert] = _('Google Cloud authorizations required')
redirect_uri = session[:error_uri]
##
# on success, the `code` param is returned
elsif params[:code]
token, expires_at = GoogleApi::CloudPlatform::Client
.new(nil, callback_google_api_auth_url)
.get_token(params[:code])
session[GoogleApi::CloudPlatform::Client.session_key_for_token] = token
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] = expires_at.to_s
redirect_uri = redirect_uri_from_session
end
##
# or google may just timeout
rescue ::Faraday::TimeoutError, ::Faraday::ConnectionFailed
flash[:alert] = _('Timeout connecting to the Google API. Please try again.')
##
# regardless, we redirect the user appropriately
ensure
redirect_to redirect_uri_from_session
redirect_to redirect_uri
end
private

26
app/controllers/projects/google_cloud/base_controller.rb Normal file
View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
class Projects::GoogleCloud::BaseController < Projects::ApplicationController
feature_category :google_cloud
before_action :admin_project_google_cloud!
before_action :google_oauth2_enabled!
before_action :feature_flag_enabled!
private
def admin_project_google_cloud!
access_denied! unless can?(current_user, :admin_project_google_cloud, project)
end
def google_oauth2_enabled!
config = Gitlab::Auth::OAuth::Provider.config_for('google_oauth2')
if config.app_id.blank? || config.app_secret.blank?
access_denied! 'This GitLab instance not configured for Google Oauth2.'
end
end
def feature_flag_enabled!
access_denied! unless Feature.enabled?(:incubation_5mp_google_cloud)
end
end

84
app/controllers/projects/google_cloud/service_accounts_controller.rb Normal file
View File

@ -0,0 +1,84 @@
# frozen_string_literal: true
class Projects::GoogleCloud::ServiceAccountsController < Projects::GoogleCloud::BaseController
before_action :validate_gcp_token!
def index
@google_cloud_path = project_google_cloud_index_path(project)
google_api_client = GoogleApi::CloudPlatform::Client.new(token_in_session, nil)
gcp_projects = google_api_client.list_projects
if gcp_projects.empty?
@js_data = {}.to_json
render status: :unauthorized, template: 'projects/google_cloud/errors/no_gcp_projects'
else
@js_data = {
gcpProjects: gcp_projects,
environments: project.environments,
cancelPath: project_google_cloud_index_path(project)
}.to_json
end
rescue Google::Apis::ClientError => error
handle_gcp_error(error, project)
end
def create
google_api_client = GoogleApi::CloudPlatform::Client.new(token_in_session, nil)
service_accounts_service = GoogleCloud::ServiceAccountsService.new(project)
gcp_project = params[:gcp_project]
environment = params[:environment]
generated_name = "GitLab :: #{@project.name} :: #{environment}"
generated_desc = "GitLab generated service account for project '#{@project.name}' and environment '#{environment}'"
service_account = google_api_client.create_service_account(gcp_project, generated_name, generated_desc)
service_account_key = google_api_client.create_service_account_key(gcp_project, service_account.unique_id)
service_accounts_service.add_for_project(
environment,
service_account.project_id,
service_account.to_json,
service_account_key.to_json
)
redirect_to project_google_cloud_index_path(project), notice: _('Service account generated successfully')
rescue Google::Apis::ClientError, Google::Apis::ServerError, Google::Apis::AuthorizationError => error
handle_gcp_error(error, project)
end
private
def validate_gcp_token!
is_token_valid = GoogleApi::CloudPlatform::Client.new(token_in_session, nil)
.validate_token(expires_at_in_session)
return if is_token_valid
return_url = project_google_cloud_service_accounts_path(project)
state = generate_session_key_redirect(request.url, return_url)
@authorize_url = GoogleApi::CloudPlatform::Client.new(nil,
callback_google_api_auth_url,
state: state).authorize_url
redirect_to @authorize_url
end
def generate_session_key_redirect(uri, error_uri)
GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key|
session[key] = uri
session[:error_uri] = error_uri
end
end
def token_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_token]
end
def expires_at_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at]
end
def handle_gcp_error(error, project)
Gitlab::ErrorTracking.track_exception(error, project_id: project.id)
@js_data = { error: error.to_s }.to_json
render status: :unauthorized, template: 'projects/google_cloud/errors/gcp_error'
end
end

27
app/controllers/projects/google_cloud_controller.rb
View File

@ -1,34 +1,11 @@
# frozen_string_literal: true
class Projects::GoogleCloudController < Projects::ApplicationController
feature_category :google_cloud
before_action :admin_project_google_cloud?
before_action :google_oauth2_enabled?
before_action :feature_flag_enabled?
class Projects::GoogleCloudController < Projects::GoogleCloud::BaseController
def index
@js_data = {
serviceAccounts: GoogleCloud::ServiceAccountsService.new(project).find_for_project,
createServiceAccountUrl: '#mocked-url-create-service',
createServiceAccountUrl: project_google_cloud_service_accounts_path(project),
emptyIllustrationUrl: ActionController::Base.helpers.image_path('illustrations/pipelines_empty.svg')
}.to_json
end
private
def admin_project_google_cloud?
access_denied! unless can?(current_user, :admin_project_google_cloud, project)
end
def google_oauth2_enabled?
config = Gitlab::Auth::OAuth::Provider.config_for('google_oauth2')
if config.app_id.blank? || config.app_secret.blank?
access_denied! 'This GitLab instance not configured for Google Oauth2.'
end
end
def feature_flag_enabled?
access_denied! unless Feature.enabled?(:incubation_5mp_google_cloud)
end
end

15
app/helpers/blame_helper.rb
View File

@ -1,6 +1,13 @@
# frozen_string_literal: true
module BlameHelper
BODY_FONT_SIZE = "0.875rem"
COMMIT_LINE_HEIGHT = 3 # 150% * 2 lines of text
COMMIT_PADDING = "10px" # 5px from both top and bottom
COMMIT_BLOCK_HEIGHT_EXP = "(#{BODY_FONT_SIZE} * #{COMMIT_LINE_HEIGHT}) + #{COMMIT_PADDING}"
CODE_LINE_HEIGHT = 1.1875
CODE_PADDING = "20px" # 10px from both top and bottom
def age_map_duration(blame_groups, project)
now = Time.zone.now
start_date = blame_groups.map { |blame_group| blame_group[:commit].committed_date }
@ -24,4 +31,12 @@ module BlameHelper
"blame-commit-age-#{age_group}"
end
end
def intrinsic_row_css(line_count)
# using rems here because the size of the row depends on the text size
# which can be customized via user agent styles and browser preferences
total_line_height_exp = "#{line_count * CODE_LINE_HEIGHT}rem + #{CODE_PADDING}"
row_height_exp = line_count == 1 ? COMMIT_BLOCK_HEIGHT_EXP : total_line_height_exp
"contain-intrinsic-size: 1px calc(#{row_height_exp})"
end
end

22
app/helpers/tab_helper.rb
View File

@ -63,6 +63,19 @@ module TabHelper
end
end
# Creates a <gl-badge> for use inside tabs.
#
# html_options - The html_options hash (default: {})
def gl_tab_counter_badge(count, html_options = {})
gl_badge_tag(
count,
{ size: :sm },
html_options.merge(
class: ['gl-tab-counter-badge', *html_options[:class]]
)
)
end
# Navigation link helper
#
# Returns an `li` element with an 'active' class if the supplied
@ -211,12 +224,3 @@ module TabHelper
current_page?(options)
end
end
def gl_tab_counter_badge(count, html_options = {})
badge_classes = %w[badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge]
content_tag(:span,
count,
class: [*html_options[:class], badge_classes].join(' '),
data: html_options[:data]
)
end

14
app/models/bulk_imports/entity.rb
View File

@ -104,8 +104,12 @@ class BulkImports::Entity < ApplicationRecord
end
end
def entity_type
source_type.gsub('_entity', '')
end
def pluralized_name
source_type.gsub('_entity', '').pluralize
entity_type.pluralize
end
def export_relations_url_path
@ -116,6 +120,14 @@ class BulkImports::Entity < ApplicationRecord
"#{export_relations_url_path}/download?relation=#{relation}"
end
def project?
source_type == 'project_entity'
end
def group?
source_type == 'group_entity'
end
private
def validate_parent_is_a_group

1
app/models/issue.rb
View File

@ -63,6 +63,7 @@ class Issue < ApplicationRecord
has_many :issue_assignees
has_many :issue_email_participants
has_one :email
has_many :assignees, class_name: "User", through: :issue_assignees
has_many :zoom_meetings
has_many :user_mentions, class_name: "IssueUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent

10
app/models/issue/email.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
class Issue::Email < ApplicationRecord
self.table_name = 'issue_emails'
belongs_to :issue
validates :email_message_id, uniqueness: true, presence: true, length: { maximum: 1000 }
validates :issue, presence: true, uniqueness: true
end

25
app/services/google_cloud/service_accounts_service.rb
View File

@ -27,6 +27,24 @@ module GoogleCloud
end
end
def add_for_project(environment, gcp_project_id, service_account, service_account_key)
project_var_create_or_replace(
environment,
'GCP_PROJECT_ID',
gcp_project_id
)
project_var_create_or_replace(
environment,
'GCP_SERVICE_ACCOUNT',
service_account
)
project_var_create_or_replace(
environment,
'GCP_SERVICE_ACCOUNT_KEY',
service_account_key
)
end
private
def group_vars_by_environment
@ -36,5 +54,12 @@ module GoogleCloud
grouped[variable.environment_scope][variable.key] = variable.value
end
end
def project_var_create_or_replace(environment_scope, key, value)
params = { key: key, filter: { environment_scope: environment_scope } }
existing_variable = ::Ci::VariablesFinder.new(@project, params).execute.first
existing_variable.destroy if existing_variable
@project.variables.create!(key: key, value: value, environment_scope: environment_scope, protected: true)
end
end
end

2
app/views/projects/blame/show.html.haml
View File

@ -27,7 +27,7 @@
- commit_data = @blame.commit_data(blame_group[:commit])
- line_count = blame_group[:lines].count
%tr
%tr{ style: intrinsic_row_css(line_count) }
%td.blame-commit{ class: commit_data.age_map_class }
.commit
= commit_data.author_avatar

6
app/views/projects/google_cloud/errors/gcp_error.html.haml Normal file
View File

@ -0,0 +1,6 @@
- breadcrumb_title _('Google Cloud')
- page_title _('Google Cloud')
- @content_class = "limit-container-width" unless fluid_layout
#js-google-cloud-error-gcp-error{ data: @js_data }

6
app/views/projects/google_cloud/errors/no_gcp_projects.html.haml Normal file
View File

@ -0,0 +1,6 @@
- breadcrumb_title _('Google Cloud')
- page_title _('Google Cloud')
- @content_class = "limit-container-width" unless fluid_layout
#js-google-cloud-error-no-gcp-projects{ data: @js_data }

8
app/views/projects/google_cloud/service_accounts/index.html.haml Normal file
View File

@ -0,0 +1,8 @@
- add_to_breadcrumbs _('Google Cloud'), @google_cloud_path
- breadcrumb_title _('Service Account')
- page_title _('Service Account')
- @content_class = "limit-container-width" unless fluid_layout
= form_tag project_google_cloud_service_accounts_path(@project), method: 'post' do
#js-google-cloud-service-accounts{ data: @js_data }

4
config/routes/project.rb
View File

@ -317,6 +317,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resources :google_cloud, only: [:index]
namespace :google_cloud do
resources :service_accounts, only: [:index, :create]
end
resources :environments, except: [:destroy] do
member do
post :stop

21
db/migrate/20211110092710_create_issue_emails.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
# See https://docs.gitlab.com/ee/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class CreateIssueEmails < Gitlab::Database::Migration[1.0]
enable_lock_retries!
def up
create_table :issue_emails do |t|
t.references :issue, index: true, null: false, unique: true, foreign_key: { on_delete: :cascade }
t.text :email_message_id, null: false, limit: 1000
t.index :email_message_id
end
end
def down
drop_table :issue_emails
end
end

1
db/schema_migrations/20211110092710 Normal file
View File

@ -0,0 +1 @@
f6312d56d2ac77537383c8671d73ad202fed9bb8eddba4bdb24d19dbe821cdf3

28
db/structure.sql
View File

@ -15278,6 +15278,22 @@ CREATE SEQUENCE issue_email_participants_id_seq
ALTER SEQUENCE issue_email_participants_id_seq OWNED BY issue_email_participants.id;
CREATE TABLE issue_emails (
id bigint NOT NULL,
issue_id bigint NOT NULL,
email_message_id text NOT NULL,
CONSTRAINT check_5abf3e6aea CHECK ((char_length(email_message_id) <= 1000))
);
CREATE SEQUENCE issue_emails_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE issue_emails_id_seq OWNED BY issue_emails.id;
CREATE TABLE issue_links (
id integer NOT NULL,
source_id integer NOT NULL,
@ -21565,6 +21581,8 @@ ALTER TABLE ONLY issue_customer_relations_contacts ALTER COLUMN id SET DEFAULT n
ALTER TABLE ONLY issue_email_participants ALTER COLUMN id SET DEFAULT nextval('issue_email_participants_id_seq'::regclass);
ALTER TABLE ONLY issue_emails ALTER COLUMN id SET DEFAULT nextval('issue_emails_id_seq'::regclass);
ALTER TABLE ONLY issue_links ALTER COLUMN id SET DEFAULT nextval('issue_links_id_seq'::regclass);
ALTER TABLE ONLY issue_metrics ALTER COLUMN id SET DEFAULT nextval('issue_metrics_id_seq'::regclass);
@ -23238,6 +23256,9 @@ ALTER TABLE ONLY issue_customer_relations_contacts
ALTER TABLE ONLY issue_email_participants
ADD CONSTRAINT issue_email_participants_pkey PRIMARY KEY (id);
ALTER TABLE ONLY issue_emails
ADD CONSTRAINT issue_emails_pkey PRIMARY KEY (id);
ALTER TABLE ONLY issue_links
ADD CONSTRAINT issue_links_pkey PRIMARY KEY (id);
@ -26159,6 +26180,10 @@ CREATE INDEX index_issue_customer_relations_contacts_on_contact_id ON issue_cust
CREATE UNIQUE INDEX index_issue_email_participants_on_issue_id_and_lower_email ON issue_email_participants USING btree (issue_id, lower(email));
CREATE INDEX index_issue_emails_on_email_message_id ON issue_emails USING btree (email_message_id);
CREATE INDEX index_issue_emails_on_issue_id ON issue_emails USING btree (issue_id);
CREATE INDEX index_issue_links_on_source_id ON issue_links USING btree (source_id);
CREATE UNIQUE INDEX index_issue_links_on_source_id_and_target_id ON issue_links USING btree (source_id, target_id);
@ -30992,6 +31017,9 @@ ALTER TABLE ONLY packages_packages
ALTER TABLE ONLY cluster_platforms_kubernetes
ADD CONSTRAINT fk_rails_e1e2cf841a FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
ALTER TABLE ONLY issue_emails
ADD CONSTRAINT fk_rails_e2ee00a8f7 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
ALTER TABLE ONLY vulnerability_finding_evidences
ADD CONSTRAINT fk_rails_e3205a0c65 FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;

2
doc/administration/geo/replication/datatypes.md
View File

@ -196,7 +196,7 @@ successfully, you must replicate their data using some other means.
|[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | No | Designs also require replication of LFS objects and Uploads. |
|[Package Registry](../../../user/packages/package_registry/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. |
|[Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.12) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_terraform_state_version_replication`, enabled by default. Verification was behind the feature flag `geo_terraform_state_version_verification`, which was removed in 14.0|
|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.5) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification is behind the feature flag `geo_merge_request_diff_verification`, enabled by default in 14.5.|
|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification is behind the feature flag `geo_merge_request_diff_verification`, enabled by default in 14.6.|
|[Versioned snippets](../../../user/snippets.md#versioned-snippets) | [**Yes** (13.7)](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [**Yes** (14.2)](https://gitlab.com/groups/gitlab-org/-/epics/2810) | No | Verification was implemented behind the feature flag `geo_snippet_repository_verification` in 13.11, and the feature flag was removed in 14.2. |
|[GitLab Pages](../../pages/index.md) | [**Yes** (14.3)](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_pages_deployment_replication`, enabled by default. |
|[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. |

2
doc/administration/gitaly/praefect.md
View File

@ -255,7 +255,7 @@ this, you must point Praefect to PgBouncer by setting Praefect database paramete
```ruby
praefect['database_host'] = PGBOUNCER_HOST
praefect['database_port'] = 6432
praefect['database_port'] = 5432
praefect['database_user'] = 'praefect'
praefect['database_password'] = PRAEFECT_SQL_PASSWORD
praefect['database_dbname'] = 'praefect_production'

111
doc/administration/reference_architectures/10k_users.md
View File

@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 10k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
card "**Internal Load Balancer**" as ilb #9370DB
@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster {
card "Database" as database {
collections "**PGBouncer** x3" as pgbouncer #4EA7FF
card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF
collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF
card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF
collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF
pgbouncer -[#4EA7FF]-> postgres_primary
postgres_primary .[#4EA7FF]> postgres_secondary
@ -83,31 +85,38 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
elb -[#6a9be7,norank]--> monitor
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[#32CD32,norank]--> ilb
gitlab -[#32CD32]r-> object_storage
gitlab -[#32CD32]----> redis
gitlab .[#32CD32]----> database
gitlab -[hidden]-> monitor
gitlab -[hidden]-> consul
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[#ff8dd1,norank]--> ilb
sidekiq -[#ff8dd1]r-> object_storage
sidekiq -[#ff8dd1]----> redis
sidekiq .[#ff8dd1]----> database
sidekiq -[hidden]-> monitor
sidekiq -[hidden]-> consul
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden]--> redis
ilb -[hidden]u-> consul
ilb -[hidden]u-> monitor
consul .[#e76a9b]u-> gitlab
consul .[#e76a9b]u-> sidekiq
consul .[#e76a9b]> monitor
consul .[#e76a9b]r-> monitor
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
@ -471,8 +480,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s
## Configure PostgreSQL
In this section, you'll be guided through configuring an external PostgreSQL database
to be used with GitLab.
In this section, you'll be guided through configuring a highly available PostgreSQL
cluster to be used with GitLab.
### Provide your own PostgreSQL instance
@ -488,12 +497,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL:
needs privileges to create the `gitlabhq_production` database.
1. Configure the GitLab application servers with the appropriate details.
This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails).
1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md)
with multiple read replicas is recommended.
See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for
further configuration steps.
### Standalone PostgreSQL using Omnibus GitLab
The recommended Omnibus GitLab configuration for a PostgreSQL cluster with
replication and failover requires:
- A minimum of three PostgreSQL nodes.
- A minimum of three Consul server nodes.
- A minimum of three PgBouncer nodes that track and handle primary database reads and writes.
- An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes.
- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled.
A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary.
The following IPs will be used as an example:
- `10.6.0.21`: PostgreSQL primary
@ -548,8 +570,8 @@ in the second step, do not supply the `EXTERNAL_URL` value.
1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
```ruby
# Disable all components except Patroni and Consul
roles(['patroni_role'])
# Disable all components except Patroni, PgBouncer and Consul
roles(['patroni_role', 'pgbouncer_role'])
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
@ -594,6 +616,15 @@ in the second step, do not supply the `EXTERNAL_URL` value.
# Replace 10.6.0.0/24 with Network Address
postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
# Local PgBouncer service for Database Load Balancing
pgbouncer['databases'] = {
gitlabhq_production: {
host: "127.0.0.1",
user: "pgbouncer",
password: '<pgbouncer_password_hash>'
}
}
# Set the network addresses that the exporters will listen on for monitoring
node_exporter['listen_address'] = '0.0.0.0:9100'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
@ -654,9 +685,11 @@ If the 'State' column for any node doesn't say "running", check the
</a>
</div>
## Configure PgBouncer
### Configure PgBouncer
Now that the PostgreSQL servers are all set up, let's configure PgBouncer
for tracking and handling reads/writes to the primary database.
Now that the PostgreSQL servers are all set up, let's configure PgBouncer.
The following IPs will be used as an example:
- `10.6.0.31`: PgBouncer 1
@ -1671,8 +1704,8 @@ To configure the Sidekiq nodes, on each one:
gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
## Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -1797,6 +1830,8 @@ On each node perform the following:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
# Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -2212,6 +2247,7 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 10k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
@ -2221,7 +2257,6 @@ card "Kubernetes via Helm Charts" as kubernetes {
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
@ -2249,37 +2284,33 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]-> monitor
elb -[hidden]-> sidekiq
elb -[hidden]-> support
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[hidden]--> consul
gitlab -[#32CD32]r--> object_storage
gitlab -[#32CD32,norank]----> redis
gitlab -[#32CD32]----> database
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[hidden]--> consul
sidekiq -[#ff8dd1]r--> object_storage
sidekiq -[#ff8dd1,norank]----> redis
sidekiq .[#ff8dd1]----> database
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden,norank]--> redis
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
monitor .[#7FFFD4]> consul
monitor .[#7FFFD4]-> database
monitor .[#7FFFD4]-> gitaly_cluster
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4]> ilb
monitor .[#7FFFD4,norank]u--> elb
consul .[#e76a9b]--> database
consul .[#e76a9b,norank]--> gitaly_cluster
consul .[#e76a9b]--> redis
@enduml
```

113
doc/administration/reference_architectures/25k_users.md
View File

@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 25k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
card "**Internal Load Balancer**" as ilb #9370DB
@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster {
card "Database" as database {
collections "**PGBouncer** x3" as pgbouncer #4EA7FF
card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF
collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF
card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF
collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF
pgbouncer -[#4EA7FF]-> postgres_primary
postgres_primary .[#4EA7FF]> postgres_secondary
@ -83,31 +85,38 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
elb -[#6a9be7,norank]--> monitor
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[#32CD32,norank]--> ilb
gitlab -[#32CD32]r-> object_storage
gitlab -[#32CD32]----> redis
gitlab .[#32CD32]----> database
gitlab -[hidden]-> monitor
gitlab -[hidden]-> consul
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[#ff8dd1,norank]--> ilb
sidekiq -[#ff8dd1]r-> object_storage
sidekiq -[#ff8dd1]----> redis
sidekiq .[#ff8dd1]----> database
sidekiq -[hidden]-> monitor
sidekiq -[hidden]-> consul
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden]--> redis
ilb -[hidden]u-> consul
ilb -[hidden]u-> monitor
consul .[#e76a9b]u-> gitlab
consul .[#e76a9b]u-> sidekiq
consul .[#e76a9b]> monitor
consul .[#e76a9b]r-> monitor
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
@ -474,8 +483,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s
## Configure PostgreSQL
In this section, you'll be guided through configuring an external PostgreSQL database
to be used with GitLab.
In this section, you'll be guided through configuring a highly available PostgreSQL
cluster to be used with GitLab.
### Provide your own PostgreSQL instance
@ -491,12 +500,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL:
needs privileges to create the `gitlabhq_production` database.
1. Configure the GitLab application servers with the appropriate details.
This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails).
1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md)
with multiple read replicas is recommended.
See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for
further configuration steps.
### Standalone PostgreSQL using Omnibus GitLab
The recommended Omnibus GitLab configuration for a PostgreSQL cluster with
replication and failover requires:
- A minimum of three PostgreSQL nodes.
- A minimum of three Consul server nodes.
- A minimum of three PgBouncer nodes that track and handle primary database reads and writes.
- An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes.
- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled.
A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary.
The following IPs will be used as an example:
- `10.6.0.21`: PostgreSQL primary
@ -551,8 +573,8 @@ in the second step, do not supply the `EXTERNAL_URL` value.
1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
```ruby
# Disable all components except Patroni and Consul
roles(['patroni_role'])
# Disable all components except Patroni, PgBouncer and Consul
roles(['patroni_role', 'pgbouncer_role'])
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
@ -597,6 +619,15 @@ in the second step, do not supply the `EXTERNAL_URL` value.
# Replace 10.6.0.0/24 with Network Address
postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
# Local PgBouncer service for Database Load Balancing
pgbouncer['databases'] = {
gitlabhq_production: {
host: "127.0.0.1",
user: "pgbouncer",
password: '<pgbouncer_password_hash>'
}
}
# Set the network addresses that the exporters will listen on for monitoring
node_exporter['listen_address'] = '0.0.0.0:9100'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
@ -657,9 +688,11 @@ If the 'State' column for any node doesn't say "running", check the
</a>
</div>
## Configure PgBouncer
### Configure PgBouncer
Now that the PostgreSQL servers are all set up, let's configure PgBouncer
for tracking and handling reads/writes to the primary database.
Now that the PostgreSQL servers are all set up, let's configure PgBouncer.
The following IPs will be used as an example:
- `10.6.0.31`: PgBouncer 1
@ -1677,8 +1710,8 @@ To configure the Sidekiq nodes, on each one:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
## Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -1805,6 +1838,8 @@ On each node perform the following:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
# Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -2212,16 +2247,16 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 25k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
together {
collections "**Webservice** x7" as gitlab #32CD32
collections "**Webservice** x4" as gitlab #32CD32
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
@ -2249,37 +2284,33 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]-> monitor
elb -[hidden]-> sidekiq
elb -[hidden]-> support
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[hidden]--> consul
gitlab -[#32CD32]r--> object_storage
gitlab -[#32CD32,norank]----> redis
gitlab -[#32CD32]----> database
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[hidden]--> consul
sidekiq -[#ff8dd1]r--> object_storage
sidekiq -[#ff8dd1,norank]----> redis
sidekiq .[#ff8dd1]----> database
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden,norank]--> redis
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
monitor .[#7FFFD4]> consul
monitor .[#7FFFD4]-> database
monitor .[#7FFFD4]-> gitaly_cluster
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4]> ilb
monitor .[#7FFFD4,norank]u--> elb
consul .[#e76a9b]--> database
consul .[#e76a9b,norank]--> gitaly_cluster
consul .[#e76a9b]--> redis
@enduml
```

18
doc/administration/reference_architectures/2k_users.md
View File

@ -41,6 +41,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 2k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
collections "**GitLab Rails** x3" as gitlab #32CD32
@ -1038,6 +1040,7 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 2k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
@ -1045,10 +1048,8 @@ card "Kubernetes via Helm Charts" as kubernetes {
together {
collections "**Webservice** x3" as gitlab #32CD32
collections "**Sidekiq** x2" as sidekiq #ff8dd1
card "**Supporting Services**" as support
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
card "**Gitaly**" as gitaly #FF8C00
@ -1057,7 +1058,6 @@ card "**Redis**" as redis #FF6347
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
gitlab -[#32CD32]--> gitaly
gitlab -[#32CD32]--> postgres
@ -1066,14 +1066,8 @@ gitlab -[#32CD32]--> redis
sidekiq -[#ff8dd1]--> gitaly
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> postgres
sidekiq -[#ff8dd1]---> redis
monitor .[#7FFFD4]u-> gitlab
monitor .[#7FFFD4]-> gitaly
monitor .[#7FFFD4]-> postgres
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4,norank]u--> elb
sidekiq -[#ff8dd1]--> postgres
sidekiq -[#ff8dd1]--> redis
@enduml
```

133
doc/administration/reference_architectures/3k_users.md
View File

@ -58,6 +58,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 3k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
card "**Internal Load Balancer**" as ilb #9370DB
@ -66,7 +68,10 @@ together {
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
together {
card "**Prometheus + Grafana**" as monitor #7FFFD4
collections "**Consul** x3" as consul #e76a9b
}
card "Gitaly Cluster" as gitaly_cluster {
collections "**Praefect** x3" as praefect #FF8C00
@ -79,47 +84,45 @@ card "Gitaly Cluster" as gitaly_cluster {
card "Database" as database {
collections "**PGBouncer** x3" as pgbouncer #4EA7FF
card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF
collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF
card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF
collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF
pgbouncer -[#4EA7FF]-> postgres_primary
postgres_primary .[#4EA7FF]> postgres_secondary
}
card "**Consul + Sentinel**" as consul_sentinel {
collections "**Consul** x3" as consul #e76a9b
collections "**Redis Sentinel** x3" as sentinel #e6e727
}
card "Redis" as redis {
collections "**Redis** x3" as redis_nodes #FF6347
redis_nodes <.[#FF6347]- sentinel
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
elb -[#6a9be7,norank]--> monitor
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[#32CD32,norank]--> ilb
gitlab -[#32CD32]r-> object_storage
gitlab -[#32CD32]----> redis
gitlab .[#32CD32]----> database
gitlab -[hidden]-> monitor
gitlab -[hidden]-> consul
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[#ff8dd1,norank]--> ilb
sidekiq -[#ff8dd1]r-> object_storage
sidekiq -[#ff8dd1]----> redis
sidekiq .[#ff8dd1]----> database
sidekiq -[hidden]-> monitor
sidekiq -[hidden]-> consul
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden]--> redis
ilb -[hidden]u-> consul
ilb -[hidden]u-> monitor
consul .[#e76a9b]u-> gitlab
consul .[#e76a9b]u-> sidekiq
consul .[#e76a9b]> monitor
consul .[#e76a9b]r-> monitor
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
@ -769,8 +772,8 @@ run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s
## Configure PostgreSQL
In this section, you'll be guided through configuring an external PostgreSQL database
to be used with GitLab.
In this section, you'll be guided through configuring a highly available PostgreSQL
cluster to be used with GitLab.
### Provide your own PostgreSQL instance
@ -786,12 +789,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL:
needs privileges to create the `gitlabhq_production` database.
1. Configure the GitLab application servers with the appropriate details.
This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails).
1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md)
with multiple read replicas is recommended.
See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for
further configuration steps.
### Standalone PostgreSQL using Omnibus GitLab
The recommended Omnibus GitLab configuration for a PostgreSQL cluster with
replication and failover requires:
- A minimum of three PostgreSQL nodes.
- A minimum of three Consul server nodes.
- A minimum of three PgBouncer nodes that track and handle primary database reads and writes.
- An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes.
- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled.
A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary.
The following IPs will be used as an example:
- `10.6.0.31`: PostgreSQL primary
@ -846,8 +862,8 @@ in the second step, do not supply the `EXTERNAL_URL` value.
1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
```ruby
# Disable all components except Patroni and Consul
roles(['patroni_role'])
# Disable all components except Patroni, PgBouncer and Consul
roles(['patroni_role', 'pgbouncer_role'])
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
@ -892,6 +908,15 @@ in the second step, do not supply the `EXTERNAL_URL` value.
# Replace 10.6.0.0/24 with Network Address
postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
# Local PgBouncer service for Database Load Balancing
pgbouncer['databases'] = {
gitlabhq_production: {
host: "127.0.0.1",
user: "pgbouncer",
password: '<pgbouncer_password_hash>'
}
}
# Set the network addresses that the exporters will listen on for monitoring
node_exporter['listen_address'] = '0.0.0.0:9100'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
@ -952,9 +977,11 @@ If the 'State' column for any node doesn't say "running", check the
</a>
</div>
## Configure PgBouncer
### Configure PgBouncer
Now that the PostgreSQL servers are all set up, let's configure PgBouncer
for tracking and handling reads/writes to the primary database.
Now that the PostgreSQL servers are all set up, let's configure PgBouncer.
The following IPs will be used as an example:
- `10.6.0.21`: PgBouncer 1
@ -1613,8 +1640,8 @@ To configure the Sidekiq nodes, one each one:
gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs
## Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -1773,6 +1800,8 @@ On each node perform the following:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs
# Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -2183,25 +2212,21 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 3k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
together {
collections "**Webservice** x2" as gitlab #32CD32
collections "**Sidekiq** x3" as sidekiq #ff8dd1
collections "**Webservice** x4" as gitlab #32CD32
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
card "**Internal Load Balancer**" as ilb #9370DB
card "**Consul + Sentinel**" as consul_sentinel {
collections "**Consul** x3" as consul #e76a9b
collections "**Redis Sentinel** x3" as sentinel #e6e727
}
collections "**Consul** x3" as consul #e76a9b
card "Gitaly Cluster" as gitaly_cluster {
collections "**Praefect** x3" as praefect #FF8C00
@ -2221,41 +2246,33 @@ card "Database" as database {
postgres_primary .[#4EA7FF]> postgres_secondary
}
card "Redis" as redis {
card "redis" as redis {
collections "**Redis** x3" as redis_nodes #FF6347
redis_nodes <.[#FF6347]- sentinel
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]-> monitor
elb -[hidden]-> sidekiq
elb -[hidden]-> support
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[hidden]--> consul
gitlab -[#32CD32]r--> object_storage
gitlab -[#32CD32,norank]----> redis
gitlab -[#32CD32]----> database
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[hidden]--> consul
sidekiq -[#ff8dd1]r--> object_storage
sidekiq -[#ff8dd1,norank]----> redis
sidekiq .[#ff8dd1]----> database
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden,norank]--> redis
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
monitor .[#7FFFD4]> consul
monitor .[#7FFFD4]-> database
monitor .[#7FFFD4]-> gitaly_cluster
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4]> ilb
monitor .[#7FFFD4,norank]u--> elb
consul .[#e76a9b]--> database
consul .[#e76a9b,norank]--> gitaly_cluster
consul .[#e76a9b]--> redis
@enduml
```

115
doc/administration/reference_architectures/50k_users.md
View File

@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 50k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
card "**Internal Load Balancer**" as ilb #9370DB
@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster {
card "Database" as database {
collections "**PGBouncer** x3" as pgbouncer #4EA7FF
card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF
collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF
card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF
collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF
pgbouncer -[#4EA7FF]-> postgres_primary
postgres_primary .[#4EA7FF]> postgres_secondary
@ -83,31 +85,38 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
elb -[#6a9be7,norank]--> monitor
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[#32CD32,norank]--> ilb
gitlab -[#32CD32]r-> object_storage
gitlab -[#32CD32]----> redis
gitlab .[#32CD32]----> database
gitlab -[hidden]-> monitor
gitlab -[hidden]-> consul
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[#ff8dd1,norank]--> ilb
sidekiq -[#ff8dd1]r-> object_storage
sidekiq -[#ff8dd1]----> redis
sidekiq .[#ff8dd1]----> database
sidekiq -[hidden]-> monitor
sidekiq -[hidden]-> consul
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden]--> redis
ilb -[hidden]u-> consul
ilb -[hidden]u-> monitor
consul .[#e76a9b]u-> gitlab
consul .[#e76a9b]u-> sidekiq
consul .[#e76a9b]> monitor
consul .[#e76a9b]r-> monitor
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
@ -480,8 +489,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s
## Configure PostgreSQL
In this section, you'll be guided through configuring an external PostgreSQL database
to be used with GitLab.
In this section, you'll be guided through configuring a highly available PostgreSQL
cluster to be used with GitLab.
### Provide your own PostgreSQL instance
@ -497,12 +506,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL:
needs privileges to create the `gitlabhq_production` database.
1. Configure the GitLab application servers with the appropriate details.
This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails).
1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md)
with multiple read replicas is recommended.
See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for
further configuration steps.
### Standalone PostgreSQL using Omnibus GitLab
The recommended Omnibus GitLab configuration for a PostgreSQL cluster with
replication and failover requires:
- A minimum of three PostgreSQL nodes.
- A minimum of three Consul server nodes.
- A minimum of three PgBouncer nodes that track and handle primary database reads and writes.
- An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes.
- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled.
A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary.
The following IPs will be used as an example:
- `10.6.0.21`: PostgreSQL primary
@ -557,8 +579,8 @@ in the second step, do not supply the `EXTERNAL_URL` value.
1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
```ruby
# Disable all components except Patroni and Consul
roles(['patroni_role'])
# Disable all components except Patroni, PgBouncer and Consul
roles(['patroni_role', 'pgbouncer_role'])
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
@ -604,6 +626,15 @@ in the second step, do not supply the `EXTERNAL_URL` value.
# Replace 10.6.0.0/24 with Network Address
postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
# Local PgBouncer service for Database Load Balancing
pgbouncer['databases'] = {
gitlabhq_production: {
host: "127.0.0.1",
user: "pgbouncer",
password: '<pgbouncer_password_hash>'
}
}
# Set the network addresses that the exporters will listen on for monitoring
node_exporter['listen_address'] = '0.0.0.0:9100'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
@ -664,9 +695,11 @@ If the 'State' column for any node doesn't say "running", check the
</a>
</div>
## Configure PgBouncer
### Configure PgBouncer
Now that the PostgreSQL servers are all set up, let's configure PgBouncer
for tracking and handling reads/writes to the primary database.
Now that the PostgreSQL servers are all set up, let's configure PgBouncer.
The following IPs will be used as an example:
- `10.6.0.31`: PgBouncer 1
@ -891,7 +924,7 @@ a node and change its status from primary to replica (and vice versa).
package of your choice. Be sure to both follow _only_ installation steps 1 and 2
on the page, and to select the correct Omnibus GitLab package, with the same version
and type (Community or Enterprise editions) as your current install.
1. Edit `/etc/gitlab/gitlab.rb` and add the same contents as the priimary node in the previous section by replacing `redis_master_node` with `redis_replica_node`:
1. Edit `/etc/gitlab/gitlab.rb` and add the same contents as the primary node in the previous section by replacing `redis_master_node` with `redis_replica_node`:
```ruby
# Specify server role as 'redis_replica_role' with Sentinel and enable Consul agent
@ -1684,8 +1717,8 @@ To configure the Sidekiq nodes, on each one:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
## Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -1819,6 +1852,8 @@ On each node perform the following:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs
# Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -2226,16 +2261,16 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 50k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
together {
collections "**Webservice** x16" as gitlab #32CD32
collections "**Webservice** x4" as gitlab #32CD32
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
@ -2263,37 +2298,33 @@ card "Database" as database {
card "redis" as redis {
collections "**Redis Persistent** x3" as redis_persistent #FF6347
collections "**Redis Cache** x3" as redis_cache #FF6347
redis_cache -[hidden]-> redis_persistent
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]-> monitor
elb -[hidden]-> sidekiq
elb -[hidden]-> support
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[hidden]--> consul
gitlab -[#32CD32]r--> object_storage
gitlab -[#32CD32,norank]----> redis
gitlab -[#32CD32]----> database
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[hidden]--> consul
sidekiq -[#ff8dd1]r--> object_storage
sidekiq -[#ff8dd1,norank]----> redis
sidekiq .[#ff8dd1]----> database
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden,norank]--> redis
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
monitor .[#7FFFD4]> consul
monitor .[#7FFFD4]-> database
monitor .[#7FFFD4]-> gitaly_cluster
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4]> ilb
monitor .[#7FFFD4,norank]u--> elb
consul .[#e76a9b]--> database
consul .[#e76a9b,norank]--> gitaly_cluster
consul .[#e76a9b]--> redis
@enduml
```

133
doc/administration/reference_architectures/5k_users.md
View File

@ -55,6 +55,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 5k
skinparam linetype ortho
card "**External Load Balancer**" as elb #6a9be7
card "**Internal Load Balancer**" as ilb #9370DB
@ -63,7 +65,10 @@ together {
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
together {
card "**Prometheus + Grafana**" as monitor #7FFFD4
collections "**Consul** x3" as consul #e76a9b
}
card "Gitaly Cluster" as gitaly_cluster {
collections "**Praefect** x3" as praefect #FF8C00
@ -76,47 +81,45 @@ card "Gitaly Cluster" as gitaly_cluster {
card "Database" as database {
collections "**PGBouncer** x3" as pgbouncer #4EA7FF
card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF
collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF
card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF
collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF
pgbouncer -[#4EA7FF]-> postgres_primary
postgres_primary .[#4EA7FF]> postgres_secondary
}
card "**Consul + Sentinel**" as consul_sentinel {
collections "**Consul** x3" as consul #e76a9b
collections "**Redis Sentinel** x3" as sentinel #e6e727
}
card "Redis" as redis {
collections "**Redis** x3" as redis_nodes #FF6347
redis_nodes <.[#FF6347]- sentinel
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]--> monitor
elb -[#6a9be7,norank]--> monitor
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[#32CD32,norank]--> ilb
gitlab -[#32CD32]r-> object_storage
gitlab -[#32CD32]----> redis
gitlab .[#32CD32]----> database
gitlab -[hidden]-> monitor
gitlab -[hidden]-> consul
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[#ff8dd1,norank]--> ilb
sidekiq -[#ff8dd1]r-> object_storage
sidekiq -[#ff8dd1]----> redis
sidekiq .[#ff8dd1]----> database
sidekiq -[hidden]-> monitor
sidekiq -[hidden]-> consul
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden]--> redis
ilb -[hidden]u-> consul
ilb -[hidden]u-> monitor
consul .[#e76a9b]u-> gitlab
consul .[#e76a9b]u-> sidekiq
consul .[#e76a9b]> monitor
consul .[#e76a9b]r-> monitor
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
@ -760,8 +763,8 @@ run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s
## Configure PostgreSQL
In this section, you'll be guided through configuring an external PostgreSQL database
to be used with GitLab.
In this section, you'll be guided through configuring a highly available PostgreSQL
cluster to be used with GitLab.
### Provide your own PostgreSQL instance
@ -777,12 +780,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL:
needs privileges to create the `gitlabhq_production` database.
1. Configure the GitLab application servers with the appropriate details.
This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails).
1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md)
with multiple read replicas is recommended.
See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for
further configuration steps.
### Standalone PostgreSQL using Omnibus GitLab
The recommended Omnibus GitLab configuration for a PostgreSQL cluster with
replication and failover requires:
- A minimum of three PostgreSQL nodes.
- A minimum of three Consul server nodes.
- A minimum of three PgBouncer nodes that track and handle primary database reads and writes.
- An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes.
- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled.
A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary.
The following IPs will be used as an example:
- `10.6.0.31`: PostgreSQL primary
@ -837,8 +853,8 @@ in the second step, do not supply the `EXTERNAL_URL` value.
1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
```ruby
# Disable all components except Patroni and Consul
roles(['patroni_role'])
# Disable all components except Patroni, PgBouncer and Consul
roles(['patroni_role', 'pgbouncer_role'])
# PostgreSQL configuration
postgresql['listen_address'] = '0.0.0.0'
@ -883,6 +899,15 @@ in the second step, do not supply the `EXTERNAL_URL` value.
# Replace 10.6.0.0/24 with Network Address
postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32)
# Local PgBouncer service for Database Load Balancing
pgbouncer['databases'] = {
gitlabhq_production: {
host: "127.0.0.1",
user: "pgbouncer",
password: '<pgbouncer_password_hash>'
}
}
# Set the network addresses that the exporters will listen on for monitoring
node_exporter['listen_address'] = '0.0.0.0:9100'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
@ -943,9 +968,11 @@ If the 'State' column for any node doesn't say "running", check the
</a>
</div>
## Configure PgBouncer
### Configure PgBouncer
Now that the PostgreSQL servers are all set up, let's configure PgBouncer
for tracking and handling reads/writes to the primary database.
Now that the PostgreSQL servers are all set up, let's configure PgBouncer.
The following IPs will be used as an example:
- `10.6.0.21`: PgBouncer 1
@ -1604,8 +1631,8 @@ To configure the Sidekiq nodes, one each one:
gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_adapter'] = 'postgresql'
gitlab_rails['db_encoding'] = 'unicode'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs
## Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -1764,6 +1791,8 @@ On each node perform the following:
gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP
gitlab_rails['db_port'] = 6432
gitlab_rails['db_password'] = '<postgresql_user_password>'
gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs
# Prevent database migrations from running on upgrade automatically
gitlab_rails['auto_migrate'] = false
@ -2153,25 +2182,21 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm
```plantuml
@startuml 5k
skinparam linetype ortho
card "Kubernetes via Helm Charts" as kubernetes {
card "**External Load Balancer**" as elb #6a9be7
together {
collections "**Webservice** x5" as gitlab #32CD32
collections "**Sidekiq** x3" as sidekiq #ff8dd1
collections "**Webservice** x4" as gitlab #32CD32
collections "**Sidekiq** x4" as sidekiq #ff8dd1
}
card "**Prometheus + Grafana**" as monitor #7FFFD4
card "**Supporting Services**" as support
}
card "**Internal Load Balancer**" as ilb #9370DB
card "**Consul + Sentinel**" as consul_sentinel {
collections "**Consul** x3" as consul #e76a9b
collections "**Redis Sentinel** x3" as sentinel #e6e727
}
collections "**Consul** x3" as consul #e76a9b
card "Gitaly Cluster" as gitaly_cluster {
collections "**Praefect** x3" as praefect #FF8C00
@ -2191,41 +2216,33 @@ card "Database" as database {
postgres_primary .[#4EA7FF]> postgres_secondary
}
card "Redis" as redis {
card "redis" as redis {
collections "**Redis** x3" as redis_nodes #FF6347
redis_nodes <.[#FF6347]- sentinel
}
cloud "**Object Storage**" as object_storage #white
elb -[#6a9be7]-> gitlab
elb -[#6a9be7]-> monitor
elb -[hidden]-> sidekiq
elb -[hidden]-> support
gitlab -[#32CD32]--> ilb
gitlab -[#32CD32]-> object_storage
gitlab -[#32CD32]---> redis
gitlab -[hidden]--> consul
gitlab -[#32CD32]r--> object_storage
gitlab -[#32CD32,norank]----> redis
gitlab -[#32CD32]----> database
sidekiq -[#ff8dd1]--> ilb
sidekiq -[#ff8dd1]-> object_storage
sidekiq -[#ff8dd1]---> redis
sidekiq -[hidden]--> consul
sidekiq -[#ff8dd1]r--> object_storage
sidekiq -[#ff8dd1,norank]----> redis
sidekiq .[#ff8dd1]----> database
ilb -[#9370DB]-> gitaly_cluster
ilb -[#9370DB]-> database
ilb -[#9370DB]--> gitaly_cluster
ilb -[#9370DB]--> database
ilb -[hidden,norank]--> redis
consul .[#e76a9b]-> database
consul .[#e76a9b]-> gitaly_cluster
consul .[#e76a9b,norank]--> redis
monitor .[#7FFFD4]> consul
monitor .[#7FFFD4]-> database
monitor .[#7FFFD4]-> gitaly_cluster
monitor .[#7FFFD4,norank]--> redis
monitor .[#7FFFD4]> ilb
monitor .[#7FFFD4,norank]u--> elb
consul .[#e76a9b]--> database
consul .[#e76a9b,norank]--> gitaly_cluster
consul .[#e76a9b]--> redis
@enduml
```

5
doc/development/graphql_guide/authorization.md
View File

@ -43,6 +43,11 @@ such as short pages, which can expose the presence of confidential resources.
See [`authorization_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/features/authorization_spec.rb)
for examples of all the authorization schemes discussed here.
<!--
NOTE: if you change this heading (or the location to this file), make sure to update
the referenced link in rubocop/cop/graphql/authorize_types.rb
-->
## Type authorization
Authorize a type by passing an ability to the `authorize` method. All

44
lib/bulk_imports/common/pipelines/badges_pipeline.rb Normal file
View File

@ -0,0 +1,44 @@
# frozen_string_literal: true
module BulkImports
module Common
module Pipelines
class BadgesPipeline
include Pipeline
extractor BulkImports::Common::Extractors::RestExtractor,
query: BulkImports::Common::Rest::GetBadgesQuery
transformer Common::Transformers::ProhibitedAttributesTransformer
def transform(context, data)
return if data.blank?
# Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project.
return if context.entity.project? && group_badge?(data)
{
name: data['name'],
link_url: data['link_url'],
image_url: data['image_url']
}
end
def load(context, data)
return if data.blank?
if context.entity.project?
context.portable.project_badges.create!(data)
else
context.portable.badges.create!(data)
end
end
private
def group_badge?(data)
data['kind'] == 'group'
end
end
end
end
end

5
lib/bulk_imports/groups/rest/get_badges_query.rb → lib/bulk_imports/common/rest/get_badges_query.rb
View File

@ -1,16 +1,17 @@
# frozen_string_literal: true
module BulkImports
module Groups
module Common
module Rest
module GetBadgesQuery
extend self
def to_h(context)
resource = context.entity.pluralized_name
encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path)
{
resource: ['groups', encoded_full_path, 'badges'].join('/'),
resource: [resource, encoded_full_path, 'badges'].join('/'),
query: {
page: context.tracker.next_page
}

32
lib/bulk_imports/groups/pipelines/badges_pipeline.rb
View File

@ -1,32 +0,0 @@
# frozen_string_literal: true
module BulkImports
module Groups
module Pipelines
class BadgesPipeline
include Pipeline
extractor BulkImports::Common::Extractors::RestExtractor,
query: BulkImports::Groups::Rest::GetBadgesQuery
transformer Common::Transformers::ProhibitedAttributesTransformer
def transform(_, data)
return if data.blank?
{
name: data['name'],
link_url: data['link_url'],
image_url: data['image_url']
}
end
def load(context, data)
return if data.blank?
context.group.badges.create!(data)
end
end
end
end
end

2
lib/bulk_imports/groups/stage.rb
View File

@ -32,7 +32,7 @@ module BulkImports
stage: 1
},
badges: {
pipeline: BulkImports::Groups::Pipelines::BadgesPipeline,
pipeline: BulkImports::Common::Pipelines::BadgesPipeline,
stage: 1
},
boards: {

4
lib/bulk_imports/projects/stage.rb
View File

@ -23,6 +23,10 @@ module BulkImports
pipeline: BulkImports::Common::Pipelines::MilestonesPipeline,
stage: 2
},
badges: {
pipeline: BulkImports::Common::Pipelines::BadgesPipeline,
stage: 2
},
issues: {
pipeline: BulkImports::Projects::Pipelines::IssuesPipeline,
stage: 3

37
lib/gitlab/database/count/reltuples_count_strategy.rb
View File

@ -32,12 +32,12 @@ module Gitlab
# Models using single-type inheritance (STI) don't work with
# reltuple count estimates. We just have to ignore them and
# use another strategy to compute them.
def non_sti_models
def non_sti_models(models)
models.reject { |model| sti_model?(model) }
end
def non_sti_table_names
non_sti_models.map(&:table_name)
def non_sti_table_names(models)
non_sti_models(models).map(&:table_name)
end
def sti_model?(model)
@ -45,21 +45,34 @@ module Gitlab
model.base_class != model
end
def table_names
models.map(&:table_name)
def table_to_model_mapping
@table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model }
end
def table_to_model(table_name)
table_to_model_mapping[table_name]
end
def size_estimates(check_statistics: true)
table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model }
results = {}
# Querying tuple stats only works on the primary. Due to load balancing, the
# easiest way to do this is to start a transaction.
ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases
get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data|
model = table_to_model[row.table_name]
data[model] = row.estimate
models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db|
base_model = Gitlab::Database.database_base_models[db_name]
tables = non_sti_table_names(models_for_db)
# Querying tuple stats only works on the primary. Due to load balancing, the
# easiest way to do this is to start a transaction.
base_model.transaction do
Gitlab::Database::SharedModel.using_connection(base_model.connection) do
get_statistics(tables, check_statistics: check_statistics).each do |row|
model = table_to_model(row.table_name)
results[model] = row.estimate
end
end
end
end
results
end
# Generates the PostgreSQL query to return the tuples for tables

2
lib/gitlab/database/count/tablesample_count_strategy.rb
View File

@ -61,7 +61,7 @@ module Gitlab
#{where_clause(model)}
SQL
rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases
rows = model.connection.select_all(query)
Integer(rows.first['count'])
end

1
lib/gitlab/database/gitlab_schemas.yml
View File

@ -261,6 +261,7 @@ issuable_severities: :gitlab_main
issuable_slas: :gitlab_main
issue_assignees: :gitlab_main
issue_customer_relations_contacts: :gitlab_main
issue_emails: :gitlab_main
issue_email_participants: :gitlab_main
issue_links: :gitlab_main
issue_metrics: :gitlab_main

4
lib/gitlab/database/migrations/background_migration_helpers.rb
View File

@ -258,7 +258,9 @@ module Gitlab
# We keep track of the estimated number of tuples to reason later
# about the overall progress of a migration.
migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do
Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
end
migration.save!
migration

2
lib/gitlab/database/pg_class.rb
View File

@ -2,7 +2,7 @@
module Gitlab
module Database
class PgClass < ActiveRecord::Base
class PgClass < SharedModel
self.table_name = 'pg_class'
def self.for_table(relname)

54
lib/gitlab/email/handler/service_desk_handler.rb
View File

@ -32,11 +32,11 @@ module Gitlab
def execute
raise ProjectNotFound if project.nil?
create_issue!
create_issue_or_note
if from_address
add_email_participant
send_thank_you_email
send_thank_you_email unless reply_email?
end
end
@ -82,6 +82,14 @@ module Gitlab
project.present? && slug == project.full_path_slug
end
def create_issue_or_note
if reply_email?
create_note_from_reply_email
else
create_issue!
end
end
def create_issue!
@issue = ::Issues::CreateService.new(
project: project,
@ -97,9 +105,33 @@ module Gitlab
raise InvalidIssueError unless @issue.persisted?
if service_desk_setting&.issue_template_missing?
create_template_not_found_note(@issue)
begin
::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id)
rescue StandardError => e
Gitlab::ErrorTracking.log_exception(e)
end
if service_desk_setting&.issue_template_missing?
create_template_not_found_note
end
end
def issue_from_reply_to
strong_memoize(:issue_from_reply_to) do
next unless mail.in_reply_to
Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue
end
end
def reply_email?
issue_from_reply_to.present?
end
def create_note_from_reply_email
@issue = issue_from_reply_to
create_note(message_including_reply)
end
def send_thank_you_email
@ -124,7 +156,7 @@ module Gitlab
end
end
def create_template_not_found_note(issue)
def create_template_not_found_note
issue_template_key = service_desk_setting&.issue_template_key
warning_note = <<-MD.strip_heredoc
@ -132,15 +164,15 @@ module Gitlab
Please check service desk settings and update the file to be used.
MD
note_params = {
noteable: issue,
note: warning_note
}
create_note(warning_note)
end
def create_note(note)
::Notes::CreateService.new(
project,
User.support_bot,
note_params
noteable: @issue,
note: note
).execute
end
@ -157,6 +189,8 @@ module Gitlab
end
def add_email_participant
return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project)
@issue.issue_email_participants.create(email: from_address)
end
end

1
lib/gitlab/import_export/project/import_export.yml
View File

@ -747,6 +747,7 @@ excluded_attributes:
- :service_desk_reply_to
- :upvotes_count
- :work_item_type_id
- :email_message_id
merge_request: &merge_request_excluded_definition
- :milestone_id
- :sprint_id

47
lib/google_api/cloud_platform/client.rb
View File

@ -1,10 +1,12 @@
# frozen_string_literal: true
require 'securerandom'
require 'google/apis/compute_v1'
require 'google/apis/container_v1'
require 'google/apis/container_v1beta1'
require 'google/apis/cloudbilling_v1'
require 'google/apis/cloudresourcemanager_v1'
require 'google/apis/iam_v1'
module GoogleApi
module CloudPlatform
@ -83,6 +85,51 @@ module GoogleApi
m[1] if m
end
def list_projects
result = []
service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new
service.authorization = access_token
response = service.fetch_all(items: :projects) do |token|
service.list_projects
end
# Google API results are paged by default, so we need to iterate through
response.each do |project|
result.append(project)
end
result
end
def create_service_account(gcp_project_id, display_name, description)
name = "projects/#{gcp_project_id}"
# initialize google iam service
service = Google::Apis::IamV1::IamService.new
service.authorization = access_token
# generate account id
random_account_id = "gitlab-" + SecureRandom.hex(11)
body_params = { account_id: random_account_id,
service_account: { display_name: display_name,
description: description } }
request_body = Google::Apis::IamV1::CreateServiceAccountRequest.new(**body_params)
service.create_service_account(name, request_body)
end
def create_service_account_key(gcp_project_id, service_account_id)
service = Google::Apis::IamV1::IamService.new
service.authorization = access_token
name = "projects/#{gcp_project_id}/serviceAccounts/#{service_account_id}"
request_body = Google::Apis::IamV1::CreateServiceAccountKeyRequest.new
service.create_service_account_key(name, request_body)
end
private
def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons)

6
locale/gitlab.pot
View File

@ -16212,6 +16212,9 @@ msgstr ""
msgid "Google Cloud Project"
msgstr ""
msgid "Google Cloud authorizations required"
msgstr ""
msgid "Google authentication is not %{link_start}properly configured%{link_end}. Ask your GitLab administrator if you want to use this service."
msgstr ""
@ -31628,6 +31631,9 @@ msgstr ""
msgid "Service URL"
msgstr ""
msgid "Service account generated successfully"
msgstr ""
msgid "Service ping is disabled in your configuration file, and cannot be enabled through this form."
msgstr ""

2
rubocop/cop/graphql/authorize_types.rb
View File

@ -5,7 +5,7 @@ module RuboCop
module Graphql
class AuthorizeTypes < RuboCop::Cop::Cop
MSG = 'Add an `authorize :ability` call to the type: '\
'https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#type-authorization'
'https://docs.gitlab.com/ee/development/graphql_guide/authorization.html#type-authorization'
# We want to exclude our own basetypes and scalars
ALLOWED_TYPES = %w[BaseEnum BaseEdge BaseScalar BasePermissionType MutationType SubscriptionType

7
scripts/review_apps/review-apps.sh
View File

@ -147,6 +147,13 @@ function disable_sign_ups() {
fi
}
function create_sample_projects() {
local create_sample_projects_rb="root_user = User.find_by_username('root'); 1.times { |i| params = { namespace_id: root_user.namespace.id, name: 'sample-project' + i.to_s, path: 'sample-project' + i.to_s, template_name: 'sample' }; ::Projects::CreateFromTemplateService.new(root_user, params).execute }"
# Queue jobs to create sample projects for root user namespace from sample data project template
retry "run_task \"${create_sample_projects_rb}\""
}
function check_kube_domain() {
echoinfo "Checking that Kube domain exists..." true

21
spec/controllers/google_api/authorizations_controller_spec.rb
View File

@ -88,5 +88,26 @@ RSpec.describe GoogleApi::AuthorizationsController do
it_behaves_like 'access denied'
end
context 'user logs in but declines authorizations' do
subject { get :callback, params: { error: 'xxx', state: state } }
let(:session_key) { 'session-key' }
let(:redirect_uri) { 'example.com' }
let(:error_uri) { 'error.com' }
let(:state) { session_key }
before do
session[GoogleApi::CloudPlatform::Client.session_key_for_redirect_uri(session_key)] = redirect_uri
session[:error_uri] = error_uri
allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |instance|
allow(instance).to receive(:get_token).and_return([token, expires_at])
end
end
it 'redirects to error uri' do
expect(subject).to redirect_to(error_uri)
end
end
end
end

2
spec/controllers/projects/raw_controller_spec.rb
View File

@ -101,7 +101,7 @@ RSpec.describe Projects::RawController do
expect(response).to have_gitlab_http_status(:too_many_requests)
end
it 'logs the event on auth.log' do
it 'logs the event on auth.log', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/345889' do
attributes = {
message: 'Application_Rate_Limiter_Request',
env: :raw_blob_request_limit,

1
spec/db/schema_spec.rb
View File

@ -53,6 +53,7 @@ RSpec.describe 'Database schema' do
identities: %w[user_id],
import_failures: %w[project_id],
issues: %w[last_edited_by_id state_id],
issue_emails: %w[email_message_id],
jira_tracker_data: %w[jira_issue_transition_id],
keys: %w[user_id],
label_links: %w[target_id],

8
spec/factories/issue_emails.rb Normal file
View File

@ -0,0 +1,8 @@
# frozen_string_literal: true
FactoryBot.define do
factory :issue_email, class: 'Issue::Email' do
issue
email_message_id { generate(:short_text) }
end
end

1
spec/factories/sequences.rb
View File

@ -21,4 +21,5 @@ FactoryBot.define do
sequence(:jira_branch) { |n| "feature/PROJ-#{n}" }
sequence(:job_name) { |n| "job #{n}" }
sequence(:work_item_type_name) { |n| "bug#{n}" }
sequence(:short_text) { |n| "someText#{n}" }
end

2
spec/features/groups/import_export/import_file_spec.rb
View File

@ -85,7 +85,7 @@ RSpec.describe 'Import/Export - Group Import', :js do
context 'when the user uploads an invalid export file' do
let(:file) { File.join(Rails.root, 'spec', %w[fixtures big-image.png]) }
it 'displays an error' do
it 'displays an error', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/343995' do
visit new_group_path
click_link 'Import group'

2
spec/fixtures/emails/service_desk_forwarded.eml vendored
View File

@ -8,7 +8,7 @@ Date: Thu, 13 Jun 2013 17:03:48 -0400
From: Jake the Dog <jake.g@adventuretime.ooo>
To: support@adventuretime.ooo
Delivered-To: support@adventuretime.ooo
Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=fdskbsf@mail.gmail.com>
Subject: The message subject! @all
Mime-Version: 1.0
Content-Type: text/plain;

23
spec/fixtures/emails/service_desk_reply.eml vendored Normal file
View File

@ -0,0 +1,23 @@
Return-Path: <alan@adventuretime.ooo>
Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400
Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+email-test-project_id-issue-@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700
Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
Date: Thu, 13 Jun 2013 17:03:48 -0400
From: Jake the Dog <alan@adventuretime.ooo>
To: incoming+email-test-project_id-issue-@appmail.adventuretime.ooo
Message-ID: <CAH_Wr+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
In-Reply-To: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
Subject: The message subject! @all
Mime-Version: 1.0
Content-Type: text/plain;
charset=ISO-8859-1
Content-Transfer-Encoding: 7bit
X-Sieve: CMU Sieve 2.2
X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
13 Jun 2013 14:03:48 -0700 (PDT)
X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
Service desk reply!
/label ~label2

17
spec/frontend/notes/components/__snapshots__/notes_app_spec.js.snap Normal file
View File

@ -0,0 +1,17 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`note_app when sort direction is asc shows skeleton notes after the loaded discussions 1`] = `
"<ul id=\\"notes-list\\" class=\\"notes main-notes-list timeline\\">
<noteable-discussion-stub discussion=\\"[object Object]\\" renderdifffile=\\"true\\" helppagepath=\\"\\" isoverviewtab=\\"true\\"></noteable-discussion-stub>
<skeleton-loading-container-stub></skeleton-loading-container-stub>
<discussion-filter-note-stub style=\\"display: none;\\"></discussion-filter-note-stub>
</ul>"
`;
exports[`note_app when sort direction is desc shows skeleton notes before the loaded discussions 1`] = `
"<ul id=\\"notes-list\\" class=\\"notes main-notes-list timeline\\">
<skeleton-loading-container-stub></skeleton-loading-container-stub>
<noteable-discussion-stub discussion=\\"[object Object]\\" renderdifffile=\\"true\\" helppagepath=\\"\\" isoverviewtab=\\"true\\"></noteable-discussion-stub>
<discussion-filter-note-stub style=\\"display: none;\\"></discussion-filter-note-stub>
</ul>"
`;

14
spec/frontend/notes/components/notes_app_spec.js
View File

@ -374,6 +374,9 @@ describe('note_app', () => {
beforeEach(() => {
store = createStore();
store.state.discussionSortOrder = constants.DESC;
store.state.isLoading = true;
store.state.discussions = [mockData.discussionMock];
wrapper = shallowMount(NotesApp, {
propsData,
store,
@ -386,11 +389,18 @@ describe('note_app', () => {
it('finds CommentForm before notes list', () => {
expect(getComponentOrder()).toStrictEqual([TYPE_COMMENT_FORM, TYPE_NOTES_LIST]);
});
it('shows skeleton notes before the loaded discussions', () => {
expect(wrapper.find('#notes-list').html()).toMatchSnapshot();
});
});
describe('when sort direction is asc', () => {
beforeEach(() => {
store = createStore();
store.state.isLoading = true;
store.state.discussions = [mockData.discussionMock];
wrapper = shallowMount(NotesApp, {
propsData,
store,
@ -403,6 +413,10 @@ describe('note_app', () => {
it('finds CommentForm after notes list', () => {
expect(getComponentOrder()).toStrictEqual([TYPE_NOTES_LIST, TYPE_COMMENT_FORM]);
});
it('shows skeleton notes after the loaded discussions', () => {
expect(wrapper.find('#notes-list').html()).toMatchSnapshot();
});
});
describe('when multiple draft types are present', () => {

12
spec/helpers/tab_helper_spec.rb
View File

@ -161,18 +161,24 @@ RSpec.describe TabHelper do
describe 'gl_tab_counter_badge' do
it 'creates a tab counter badge' do
expect(gl_tab_counter_badge(1)).to eq('<span class="badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge">1</span>')
expect(helper.gl_tab_counter_badge(1)).to eq(
'<span class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge">1</span>'
)
end
context 'with extra classes' do
it 'creates a tab counter badge with the correct class attribute' do
expect(gl_tab_counter_badge(1, { class: 'js-test' })).to eq('<span class="js-test badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge">1</span>')
expect(helper.gl_tab_counter_badge(1, { class: 'js-test' })).to eq(
'<span class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge js-test">1</span>'
)
end
end
context 'with data attributes' do
it 'creates a tab counter badge with the data attributes' do
expect(gl_tab_counter_badge(1, { data: { some_attribute: 'foo' } })).to eq('<span class="badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge" data-some-attribute="foo">1</span>')
expect(helper.gl_tab_counter_badge(1, { data: { some_attribute: 'foo' } })).to eq(
'<span data-some-attribute="foo" class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge">1</span>'
)
end
end
end

96
spec/lib/bulk_imports/common/pipelines/badges_pipeline_spec.rb Normal file
View File

@ -0,0 +1,96 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Common::Pipelines::BadgesPipeline do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project) }
let(:entity) { create(:bulk_import_entity, group: group) }
let(:tracker) { create(:bulk_import_tracker, entity: entity) }
let(:context) { BulkImports::Pipeline::Context.new(tracker) }
subject(:pipeline) { described_class.new(context) }
describe '#run' do
let(:first_page) { extracted_data(has_next_page: true) }
let(:last_page) { extracted_data(name: 'badge2') }
before do
allow_next_instance_of(BulkImports::Common::Extractors::RestExtractor) do |extractor|
allow(extractor).to receive(:extract).and_return(first_page, last_page)
end
end
it 'imports a group badge' do
expect { pipeline.run }.to change(Badge, :count).by(2)
badge = group.badges.last
expect(badge.name).to eq('badge2')
expect(badge.link_url).to eq(badge_data['link_url'])
expect(badge.image_url).to eq(badge_data['image_url'])
end
context 'when project entity' do
let(:first_page) { extracted_data(has_next_page: true) }
let(:last_page) { extracted_data(name: 'badge2', kind: 'project') }
let(:entity) { create(:bulk_import_entity, :project_entity, project: project) }
it 'imports a project badge & skips group badge' do
expect { pipeline.run }.to change(Badge, :count).by(1)
badge = project.badges.last
expect(badge.name).to eq('badge2')
expect(badge.link_url).to eq(badge_data['link_url'])
expect(badge.image_url).to eq(badge_data['image_url'])
expect(badge.type).to eq('ProjectBadge')
end
end
describe '#transform' do
it 'return transformed badge hash' do
badge = subject.transform(context, badge_data)
expect(badge[:name]).to eq('badge')
expect(badge[:link_url]).to eq(badge_data['link_url'])
expect(badge[:image_url]).to eq(badge_data['image_url'])
expect(badge.keys).to contain_exactly(:name, :link_url, :image_url)
end
context 'when data is blank' do
it 'does nothing when the data is blank' do
expect(subject.transform(context, nil)).to be_nil
end
end
context 'when project entity & group badge' do
let(:entity) { create(:bulk_import_entity, :project_entity, project: project) }
it 'returns' do
expect(subject.transform(context, { 'name' => 'test', 'kind' => 'group' })).to be_nil
end
end
end
def badge_data(name = 'badge', kind = 'group')
{
'name' => name,
'link_url' => 'https://gitlab.example.com',
'image_url' => 'https://gitlab.example.com/image.png',
'kind' => kind
}
end
def extracted_data(name: 'badge', kind: 'group', has_next_page: false)
page_info = {
'has_next_page' => has_next_page,
'next_page' => has_next_page ? '2' : nil
}
BulkImports::Pipeline::ExtractedData.new(data: [badge_data(name, kind)], page_info: page_info)
end
end
end

36
spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb Normal file
View File

@ -0,0 +1,36 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Common::Rest::GetBadgesQuery do
describe '.to_h' do
shared_examples 'resource and page info query' do
let(:tracker) { create(:bulk_import_tracker, entity: entity) }
let(:context) { BulkImports::Pipeline::Context.new(tracker) }
let(:encoded_full_path) { ERB::Util.url_encode(entity.source_full_path) }
it 'returns correct query and page info' do
expected = {
resource: [entity.pluralized_name, encoded_full_path, 'badges'].join('/'),
query: {
page: context.tracker.next_page
}
}
expect(described_class.to_h(context)).to eq(expected)
end
end
context 'when entity is group' do
let(:entity) { create(:bulk_import_entity) }
include_examples 'resource and page info query'
end
context 'when entity is project' do
let(:entity) { create(:bulk_import_entity, :project_entity) }
include_examples 'resource and page info query'
end
end
end

116
spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb
View File

@ -1,116 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Groups::Pipelines::BadgesPipeline do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:entity) do
create(
:bulk_import_entity,
source_full_path: 'source/full/path',
destination_name: 'My Destination Group',
destination_namespace: group.full_path,
group: group
)
end
let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) }
let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) }
subject { described_class.new(context) }
describe '#run' do
it 'imports a group badge' do
first_page = extracted_data(has_next_page: true)
last_page = extracted_data(name: 'badge2')
allow_next_instance_of(BulkImports::Common::Extractors::RestExtractor) do |extractor|
allow(extractor)
.to receive(:extract)
.and_return(first_page, last_page)
end
expect { subject.run }.to change(Badge, :count).by(2)
badge = group.badges.last
expect(badge.name).to eq('badge2')
expect(badge.link_url).to eq(badge_data['link_url'])
expect(badge.image_url).to eq(badge_data['image_url'])
end
describe '#load' do
it 'creates a badge' do
expect { subject.load(context, badge_data) }.to change(Badge, :count).by(1)
badge = group.badges.first
badge_data.each do |key, value|
expect(badge[key]).to eq(value)
end
end
it 'does nothing when the data is blank' do
expect { subject.load(context, nil) }.not_to change(Badge, :count)
end
end
describe '#transform' do
it 'return transformed badge hash' do
badge = subject.transform(context, badge_data)
expect(badge[:name]).to eq('badge')
expect(badge[:link_url]).to eq(badge_data['link_url'])
expect(badge[:image_url]).to eq(badge_data['image_url'])
expect(badge.keys).to contain_exactly(:name, :link_url, :image_url)
end
context 'when data is blank' do
it 'does nothing when the data is blank' do
expect(subject.transform(context, nil)).to be_nil
end
end
end
describe 'pipeline parts' do
it { expect(described_class).to include_module(BulkImports::Pipeline) }
it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
it 'has extractors' do
expect(described_class.get_extractor)
.to eq(
klass: BulkImports::Common::Extractors::RestExtractor,
options: {
query: BulkImports::Groups::Rest::GetBadgesQuery
}
)
end
it 'has transformers' do
expect(described_class.transformers)
.to contain_exactly(
{ klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil }
)
end
end
def badge_data(name = 'badge')
{
'name' => name,
'link_url' => 'https://gitlab.example.com',
'image_url' => 'https://gitlab.example.com/image.png'
}
end
def extracted_data(name: 'badge', has_next_page: false)
page_info = {
'has_next_page' => has_next_page,
'next_page' => has_next_page ? '2' : nil
}
BulkImports::Pipeline::ExtractedData.new(data: [badge_data(name)], page_info: page_info)
end
end
end

22
spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb
View File

@ -1,22 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Groups::Rest::GetBadgesQuery do
describe '.to_h' do
it 'returns query resource and page info' do
entity = create(:bulk_import_entity)
tracker = create(:bulk_import_tracker, entity: entity)
context = BulkImports::Pipeline::Context.new(tracker)
encoded_full_path = ERB::Util.url_encode(entity.source_full_path)
expected = {
resource: ['groups', encoded_full_path, 'badges'].join('/'),
query: {
page: context.tracker.next_page
}
}
expect(described_class.to_h(context)).to eq(expected)
end
end
end

2
spec/lib/bulk_imports/groups/stage_spec.rb
View File

@ -13,7 +13,7 @@ RSpec.describe BulkImports::Groups::Stage do
[1, BulkImports::Groups::Pipelines::MembersPipeline],
[1, BulkImports::Common::Pipelines::LabelsPipeline],
[1, BulkImports::Common::Pipelines::MilestonesPipeline],
[1, BulkImports::Groups::Pipelines::BadgesPipeline],
[1, BulkImports::Common::Pipelines::BadgesPipeline],
[2, BulkImports::Common::Pipelines::BoardsPipeline]
]
end

1
spec/lib/bulk_imports/projects/stage_spec.rb
View File

@ -11,6 +11,7 @@ RSpec.describe BulkImports::Projects::Stage do
[1, BulkImports::Projects::Pipelines::RepositoryPipeline],
[2, BulkImports::Common::Pipelines::LabelsPipeline],
[2, BulkImports::Common::Pipelines::MilestonesPipeline],
[2, BulkImports::Common::Pipelines::BadgesPipeline],
[3, BulkImports::Projects::Pipelines::IssuesPipeline],
[3, BulkImports::Projects::Pipelines::SnippetsPipeline],
[4, BulkImports::Common::Pipelines::BoardsPipeline],

16
spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb
View File

@ -5,24 +5,24 @@ require 'spec_helper'
RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do
before do
create_list(:project, 3)
create(:identity)
create_list(:ci_instance_variable, 2)
end
subject { described_class.new(models).count }
describe '#count' do
let(:models) { [Project, Identity] }
let(:models) { [Project, Ci::InstanceVariable] }
context 'when reltuples is up to date' do
before do
ActiveRecord::Base.connection.execute('ANALYZE projects')
ActiveRecord::Base.connection.execute('ANALYZE identities')
Project.connection.execute('ANALYZE projects')
Ci::InstanceVariable.connection.execute('ANALYZE ci_instance_variables')
end
it 'uses statistics to do the count' do
models.each { |model| expect(model).not_to receive(:count) }
expect(subject).to eq({ Project => 3, Identity => 1 })
expect(subject).to eq({ Project => 3, Ci::InstanceVariable => 2 })
end
end
@ -31,7 +31,7 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do
before do
models.each do |model|
ActiveRecord::Base.connection.execute("ANALYZE #{model.table_name}")
model.connection.execute("ANALYZE #{model.table_name}")
end
end
@ -45,7 +45,9 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do
context 'insufficient permissions' do
it 'returns an empty hash' do
allow(ActiveRecord::Base).to receive(:transaction).and_raise(PG::InsufficientPrivilege)
Gitlab::Database.database_base_models.each_value do |base_model|
allow(base_model).to receive(:transaction).and_raise(PG::InsufficientPrivilege)
end
expect(subject).to eq({})
end

8
spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb
View File

@ -5,11 +5,12 @@ require 'spec_helper'
RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do
before do
create_list(:project, 3)
create_list(:ci_instance_variable, 2)
create(:identity)
create(:group)
end
let(:models) { [Project, Identity, Group, Namespace] }
let(:models) { [Project, Ci::InstanceVariable, Identity, Group, Namespace] }
let(:strategy) { described_class.new(models) }
subject { strategy.count }
@ -20,7 +21,8 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do
Project => threshold + 1,
Identity => threshold - 1,
Group => threshold + 1,
Namespace => threshold + 1
Namespace => threshold + 1,
Ci::InstanceVariable => threshold + 1
}
end
@ -43,12 +45,14 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do
expect(Project).not_to receive(:count)
expect(Group).not_to receive(:count)
expect(Namespace).not_to receive(:count)
expect(Ci::InstanceVariable).not_to receive(:count)
result = subject
expect(result[Project]).to eq(3)
expect(result[Group]).to eq(1)
# 1-Group, 3 namespaces for each project and 3 project namespaces for each project
expect(result[Namespace]).to eq(7)
expect(result[Ci::InstanceVariable]).to eq(2)
end
end

127
spec/lib/gitlab/email/handler/service_desk_handler_spec.rb
View File

@ -12,6 +12,8 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
let(:email_raw) { email_fixture('emails/service_desk.eml') }
let(:author_email) { 'jake@adventuretime.ooo' }
let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com' }
let_it_be(:group) { create(:group, :private, name: "email") }
let(:expected_description) do
@ -40,6 +42,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
expect(new_issue.all_references.all).to be_empty
expect(new_issue.title).to eq("The message subject! @all")
expect(new_issue.description).to eq(expected_description.strip)
expect(new_issue.email&.email_message_id).to eq(message_id)
end
it 'creates an issue_email_participant' do
@ -72,6 +75,95 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
it_behaves_like 'a new issue request'
end
context 'when replying to issue creation email' do
def receive_reply
reply_email_raw = email_fixture('emails/service_desk_reply.eml')
second_receiver = Gitlab::Email::Receiver.new(reply_email_raw)
second_receiver.execute
end
context 'when an issue with message_id has been found' do
before do
receiver.execute
end
subject do
receive_reply
end
it 'does not create an additional issue' do
expect { subject }.not_to change { Issue.count }
end
it 'adds a comment to the created issue' do
subject
notes = Issue.last.notes
new_note = notes.first
expect(notes.count).to eq(1)
expect(new_note.note).to eq("Service desk reply!\n\n`/label ~label2`")
expect(new_note.author).to eql(User.support_bot)
end
it 'does not send thank you email' do
expect(Notify).not_to receive(:service_desk_thank_you_email)
subject
end
context 'when issue_email_participants FF is enabled' do
it 'creates 2 issue_email_participants' do
subject
expect(Issue.last.issue_email_participants.map(&:email))
.to match_array(%w(alan@adventuretime.ooo jake@adventuretime.ooo))
end
end
context 'when issue_email_participants FF is disabled' do
before do
stub_feature_flags(issue_email_participants: false)
end
it 'creates only 1 issue_email_participant' do
subject
expect(Issue.last.issue_email_participants.map(&:email))
.to match_array(%w(jake@adventuretime.ooo))
end
end
end
context 'when an issue with message_id has not been found' do
subject do
receive_reply
end
it 'creates a new issue correctly' do
expect { subject }.to change { Issue.count }.by(1)
issue = Issue.last
expect(issue.description).to eq("Service desk reply!\n\n`/label ~label2`")
end
it 'sends thank you email once' do
expect(Notify).to receive(:service_desk_thank_you_email).once.and_return(double(deliver_later: true))
subject
end
it 'creates 1 issue_email_participant' do
subject
expect(Issue.last.issue_email_participants.map(&:email))
.to match_array(%w(alan@adventuretime.ooo))
end
end
end
context 'when using issue templates' do
let_it_be(:user) { create(:user) }
@ -270,6 +362,20 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
end
end
context 'when issue email creation fails' do
before do
allow(::Issue::Email).to receive(:create!).and_raise(StandardError)
end
it 'still creates a new issue' do
expect { receiver.execute }.to change { Issue.count }.by(1)
end
it 'does not create issue email record' do
expect { receiver.execute }.not_to change { Issue::Email.count }
end
end
context 'when rate limiting is in effect', :freeze_time, :clean_gitlab_redis_rate_limiting do
let(:receiver) { Gitlab::Email::Receiver.new(email_raw) }
@ -291,19 +397,19 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
rescue RateLimitedService::RateLimitedError
end.to change { Issue.count }.by(1)
end
end
context 'when requests are sent by different users' do
let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') }
let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) }
context 'when requests are sent by different users' do
let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') }
let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) }
subject do
receiver.execute
receiver2.execute
end
subject do
receiver.execute
receiver2.execute
end
it 'creates 2 issues' do
expect { subject }.to change { Issue.count }.by(2)
end
it 'creates 2 issues' do
expect { subject }.to change { Issue.count }.by(2)
end
end
@ -389,6 +495,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do
context 'when the email is forwarded through an alias' do
let(:author_email) { 'jake.g@adventuretime.ooo' }
let(:email_raw) { email_fixture('emails/service_desk_forwarded.eml') }
let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=fdskbsf@mail.gmail.com' }
it_behaves_like 'a new issue request'
end

1
spec/lib/gitlab/import_export/all_models.yml
View File

@ -61,6 +61,7 @@ issues:
- pending_escalations
- customer_relations_contacts
- issue_customer_relations_contacts
- email
work_item_type:
- issues
events:

1
spec/lib/gitlab/import_export/safe_model_attributes.yml
View File

@ -33,6 +33,7 @@ Issue:
- health_status
- external_key
- issue_type
- email_message_id
Event:
- id
- target_type

43
spec/lib/google_api/cloud_platform/client_spec.rb
View File

@ -209,4 +209,47 @@ RSpec.describe GoogleApi::CloudPlatform::Client do
expect(subject.header).to eq({ 'User-Agent': 'GitLab/10.3 (GPN:GitLab;)' })
end
end
describe '#list_projects' do
subject { client.list_projects }
let(:list_of_projects) { [{}, {}, {}] }
let(:next_page_token) { nil }
let(:operation) { double('projects': list_of_projects, 'next_page_token': next_page_token) }
it 'calls Google Api CloudResourceManagerService#list_projects' do
expect_any_instance_of(Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService)
.to receive(:list_projects)
.and_return(operation)
is_expected.to eq(list_of_projects)
end
end
describe '#create_service_account' do
subject { client.create_service_account(spy, spy, spy) }
let(:operation) { double('Service Account') }
it 'calls Google Api IamService#create_service_account' do
expect_any_instance_of(Google::Apis::IamV1::IamService)
.to receive(:create_service_account)
.with(any_args)
.and_return(operation)
is_expected.to eq(operation)
end
end
describe '#create_service_account_key' do
subject { client.create_service_account_key(spy, spy) }
let(:operation) { double('Service Account Key') }
it 'class Google Api IamService#create_service_account_key' do
expect_any_instance_of(Google::Apis::IamV1::IamService)
.to receive(:create_service_account_key)
.with(any_args)
.and_return(operation)
is_expected.to eq(operation)
end
end
end

30
spec/models/bulk_imports/entity_spec.rb
View File

@ -252,4 +252,34 @@ RSpec.describe BulkImports::Entity, type: :model do
.to eq("/groups/#{entity.encoded_source_full_path}/export_relations/download?relation=test")
end
end
describe '#entity_type' do
it 'returns entity type' do
group_entity = build(:bulk_import_entity)
project_entity = build(:bulk_import_entity, :project_entity)
expect(group_entity.entity_type).to eq('group')
expect(project_entity.entity_type).to eq('project')
end
end
describe '#project?' do
it 'returns true if project entity' do
group_entity = build(:bulk_import_entity)
project_entity = build(:bulk_import_entity, :project_entity)
expect(group_entity.project?).to eq(false)
expect(project_entity.project?).to eq(true)
end
end
describe '#group?' do
it 'returns true if group entity' do
group_entity = build(:bulk_import_entity)
project_entity = build(:bulk_import_entity, :project_entity)
expect(group_entity.group?).to eq(true)
expect(project_entity.group?).to eq(false)
end
end
end

19
spec/models/issue/email_spec.rb Normal file
View File

@ -0,0 +1,19 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Issue::Email do
describe 'Associations' do
it { is_expected.to belong_to(:issue) }
end
describe 'Validations' do
subject { build(:issue_email) }
it { is_expected.to validate_presence_of(:issue) }
it { is_expected.to validate_uniqueness_of(:issue) }
it { is_expected.to validate_uniqueness_of(:email_message_id) }
it { is_expected.to validate_length_of(:email_message_id).is_at_most(1000) }
it { is_expected.to validate_presence_of(:email_message_id) }
end
end

1
spec/models/issue_spec.rb
View File

@ -32,6 +32,7 @@ RSpec.describe Issue do
it { is_expected.to have_and_belong_to_many(:self_managed_prometheus_alert_events) }
it { is_expected.to have_many(:prometheus_alerts) }
it { is_expected.to have_many(:issue_email_participants) }
it { is_expected.to have_one(:email) }
it { is_expected.to have_many(:timelogs).autosave(true) }
it { is_expected.to have_one(:incident_management_issuable_escalation_status) }
it { is_expected.to have_many(:issue_customer_relations_contacts) }

2
spec/requests/api/graphql/ci/jobs_spec.rb
View File

@ -90,7 +90,7 @@ RSpec.describe 'Query.project.pipeline' do
create(:ci_build_need, build: deploy_job, name: 'rspec 1 2')
end
it 'reports the build needs and previous stages with no duplicates' do
it 'reports the build needs and previous stages with no duplicates', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/346433' do
post_graphql(query, current_user: user)
expect(jobs_graphql_data).to contain_exactly(

184
spec/requests/projects/google_cloud/service_accounts_controller_spec.rb Normal file
View File

@ -0,0 +1,184 @@
# frozen_string_literal: true
require 'spec_helper'
# Mock Types
MockGoogleOAuth2Credentials = Struct.new(:app_id, :app_secret)
MockServiceAccount = Struct.new(:project_id, :unique_id)
RSpec.describe Projects::GoogleCloud::ServiceAccountsController do
let_it_be(:project) { create(:project, :public) }
describe 'GET index' do
let_it_be(:url) { "#{project_google_cloud_service_accounts_path(project)}" }
let(:user_guest) { create(:user) }
let(:user_developer) { create(:user) }
let(:user_maintainer) { create(:user) }
let(:user_creator) { project.creator }
let(:unauthorized_members) { [user_guest, user_developer] }
let(:authorized_members) { [user_maintainer, user_creator] }
before do
project.add_guest(user_guest)
project.add_developer(user_developer)
project.add_maintainer(user_maintainer)
end
context 'when a public request is made' do
it 'returns not found on GET request' do
get url
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns not found on POST request' do
post url
expect(response).to have_gitlab_http_status(:not_found)
end
end
context 'when unauthorized members make requests' do
it 'returns not found on GET request' do
unauthorized_members.each do |unauthorized_member|
sign_in(unauthorized_member)
get url
expect(response).to have_gitlab_http_status(:not_found)
end
end
it 'returns not found on POST request' do
unauthorized_members.each do |unauthorized_member|
sign_in(unauthorized_member)
post url
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
context 'when authorized members make requests' do
it 'redirects on GET request' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to redirect_to(assigns(:authorize_url))
end
end
it 'redirects on POST request' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
post url
expect(response).to redirect_to(assigns(:authorize_url))
end
end
context 'and user has successfully completed the google oauth2 flow' do
before do
allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client|
allow(client).to receive(:validate_token).and_return(true)
allow(client).to receive(:list_projects).and_return([{}, {}, {}])
allow(client).to receive(:create_service_account).and_return(MockServiceAccount.new(123, 456))
allow(client).to receive(:create_service_account_key).and_return({})
end
end
it 'returns success on GET' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to have_gitlab_http_status(:ok)
end
end
it 'returns success on POST' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
post url, params: { gcp_project: 'prj1', environment: 'env1' }
expect(response).to redirect_to(project_google_cloud_index_path(project))
end
end
end
context 'but google returns client error' do
before do
allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client|
allow(client).to receive(:validate_token).and_return(true)
allow(client).to receive(:list_projects).and_raise(Google::Apis::ClientError.new(''))
allow(client).to receive(:create_service_account).and_raise(Google::Apis::ClientError.new(''))
allow(client).to receive(:create_service_account_key).and_raise(Google::Apis::ClientError.new(''))
end
end
it 'renders gcp_error template on GET' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to render_template(:gcp_error)
end
end
it 'renders gcp_error template on POST' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
post url, params: { gcp_project: 'prj1', environment: 'env1' }
expect(response).to render_template(:gcp_error)
end
end
end
context 'but gitlab instance is not configured for google oauth2' do
before do
unconfigured_google_oauth2 = MockGoogleOAuth2Credentials.new('', '')
allow(Gitlab::Auth::OAuth::Provider).to receive(:config_for)
.with('google_oauth2')
.and_return(unconfigured_google_oauth2)
end
it 'returns forbidden' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
context 'but feature flag is disabled' do
before do
stub_feature_flags(incubation_5mp_google_cloud: false)
end
it 'returns not found' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
end
end
end

2
spec/rubocop/cop/graphql/authorize_types_spec.rb
View File

@ -11,7 +11,7 @@ RSpec.describe RuboCop::Cop::Graphql::AuthorizeTypes do
expect_offense(<<~TYPE)
module Types
class AType < BaseObject
^^^^^^^^^^^^^^^^^^^^^^^^ Add an `authorize :ability` call to the type: https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#type-authorization
^^^^^^^^^^^^^^^^^^^^^^^^ Add an `authorize :ability` call to the type: https://docs.gitlab.com/ee/development/graphql_guide/authorization.html#type-authorization
field :a_thing
field :another_thing
end

69
spec/services/google_cloud/service_accounts_service_spec.rb
View File

@ -3,11 +3,11 @@
require 'spec_helper'
RSpec.describe GoogleCloud::ServiceAccountsService do
let_it_be(:project) { create(:project) }
let(:service) { described_class.new(project) }
describe 'find_for_project' do
let_it_be(:project) { create(:project) }
context 'when a project does not have GCP service account vars' do
before do
project.variables.build(key: 'blah', value: 'foo', environment_scope: 'world')
@ -21,13 +21,13 @@ RSpec.describe GoogleCloud::ServiceAccountsService do
context 'when a project has GCP service account ci vars' do
before do
project.variables.build(environment_scope: '*', key: 'GCP_PROJECT_ID', value: 'prj1')
project.variables.build(environment_scope: '*', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock')
project.variables.build(environment_scope: 'staging', key: 'GCP_PROJECT_ID', value: 'prj2')
project.variables.build(environment_scope: 'staging', key: 'GCP_SERVICE_ACCOUNT', value: 'mock')
project.variables.build(environment_scope: 'production', key: 'GCP_PROJECT_ID', value: 'prj3')
project.variables.build(environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT', value: 'mock')
project.variables.build(environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock')
project.variables.build(protected: true, environment_scope: '*', key: 'GCP_PROJECT_ID', value: 'prj1')
project.variables.build(protected: true, environment_scope: '*', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock')
project.variables.build(protected: true, environment_scope: 'staging', key: 'GCP_PROJECT_ID', value: 'prj2')
project.variables.build(protected: true, environment_scope: 'staging', key: 'GCP_SERVICE_ACCOUNT', value: 'mock')
project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_PROJECT_ID', value: 'prj3')
project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT', value: 'mock')
project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock')
project.save!
end
@ -55,4 +55,55 @@ RSpec.describe GoogleCloud::ServiceAccountsService do
end
end
end
describe 'add_for_project' do
let_it_be(:project) { create(:project) }
it 'saves GCP creds as project CI vars' do
service.add_for_project('env_1', 'gcp_prj_id_1', 'srv_acc_1', 'srv_acc_key_1')
service.add_for_project('env_2', 'gcp_prj_id_2', 'srv_acc_2', 'srv_acc_key_2')
list = service.find_for_project
aggregate_failures 'testing list of service accounts' do
expect(list.length).to eq(2)
expect(list.first[:environment]).to eq('env_1')
expect(list.first[:gcp_project]).to eq('gcp_prj_id_1')
expect(list.first[:service_account_exists]).to eq(true)
expect(list.first[:service_account_key_exists]).to eq(true)
expect(list.second[:environment]).to eq('env_2')
expect(list.second[:gcp_project]).to eq('gcp_prj_id_2')
expect(list.second[:service_account_exists]).to eq(true)
expect(list.second[:service_account_key_exists]).to eq(true)
end
end
it 'replaces previously stored CI vars with new CI vars' do
service.add_for_project('env_1', 'new_project', 'srv_acc_1', 'srv_acc_key_1')
list = service.find_for_project
aggregate_failures 'testing list of service accounts' do
expect(list.length).to eq(2)
# asserting that the first service account is replaced
expect(list.first[:environment]).to eq('env_1')
expect(list.first[:gcp_project]).to eq('new_project')
expect(list.first[:service_account_exists]).to eq(true)
expect(list.first[:service_account_key_exists]).to eq(true)
expect(list.second[:environment]).to eq('env_2')
expect(list.second[:gcp_project]).to eq('gcp_prj_id_2')
expect(list.second[:service_account_exists]).to eq(true)
expect(list.second[:service_account_key_exists]).to eq(true)
end
end
it 'underlying project CI vars must be protected' do
expect(project.variables.first.protected).to eq(true)
expect(project.variables.second.protected).to eq(true)
end
end
end