kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2022-01-11 12:14:06 +00:00
parent 1b5a83917d
commit 22391da126
59 changed files with 882 additions and 411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app/assets/javascripts/behaviors/copy_code.js
View file

@ -30,6 +30,7 @@ class CopyCodeButton extends HTMLElement {
function addCodeButton() {
[...document.querySelectorAll('pre.code.js-syntax-highlight')]
.filter((el) => el.attr('lang') !== 'mermaid')
.filter((el) => !el.closest('.js-markdown-code'))
.forEach((el) => {
const copyCodeEl = document.createElement('copy-code');

2
app/assets/stylesheets/framework/variables_overrides.scss
View file

@ -5,7 +5,7 @@
$secondary: $gray-light;
$input-disabled-bg: $gray-light;
$input-border-color: $gray-100;
$input-border-color: $gray-400;
$input-color: $gl-text-color;
$input-font-size: $gl-font-size;
$font-family-sans-serif: $regular-font;

2
app/assets/stylesheets/startup/startup-dark.scss
View file

@ -140,7 +140,7 @@ h1 {
color: #fafafa;
background-color: #333;
background-clip: padding-box;
border: 1px solid #404040;
border: 1px solid #868686;
border-radius: 0.25rem;
}
@media (prefers-reduced-motion: reduce) {

2
app/assets/stylesheets/startup/startup-general.scss
View file

@ -121,7 +121,7 @@ h1 {
color: #303030;
background-color: #fff;
background-clip: padding-box;
border: 1px solid #dbdbdb;
border: 1px solid #868686;
border-radius: 0.25rem;
}
@media (prefers-reduced-motion: reduce) {

2
app/assets/stylesheets/startup/startup-signin.scss
View file

@ -189,7 +189,7 @@ hr {
color: #303030;
background-color: #fff;
background-clip: padding-box;
border: 1px solid #dbdbdb;
border: 1px solid #868686;
border-radius: 0.25rem;
}
@media (prefers-reduced-motion: reduce) {

35
app/controllers/projects/google_cloud/base_controller.rb
View file

@ -23,4 +23,39 @@ class Projects::GoogleCloud::BaseController < Projects::ApplicationController
def feature_flag_enabled!
access_denied! unless Feature.enabled?(:incubation_5mp_google_cloud, project)
end
def validate_gcp_token!
is_token_valid = GoogleApi::CloudPlatform::Client.new(token_in_session, nil)
.validate_token(expires_at_in_session)
return if is_token_valid
return_url = project_google_cloud_index_path(project)
state = generate_session_key_redirect(request.url, return_url)
@authorize_url = GoogleApi::CloudPlatform::Client.new(nil,
callback_google_api_auth_url,
state: state).authorize_url
redirect_to @authorize_url
end
def generate_session_key_redirect(uri, error_uri)
GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key|
session[key] = uri
session[:error_uri] = error_uri
end
end
def token_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_token]
end
def expires_at_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at]
end
def handle_gcp_error(error, project)
Gitlab::ErrorTracking.track_exception(error, project_id: project.id)
@js_data = { screen: 'gcp_error', error: error.to_s }.to_json
render status: :unauthorized, template: 'projects/google_cloud/errors/gcp_error'
end
end

13
app/controllers/projects/google_cloud/deployments_controller.rb Normal file
View file

@ -0,0 +1,13 @@
# frozen_string_literal: true
class Projects::GoogleCloud::DeploymentsController < Projects::GoogleCloud::BaseController
before_action :validate_gcp_token!
def cloud_run
render json: "Placeholder"
end
def cloud_storage
render json: "Placeholder"
end
end

37
app/controllers/projects/google_cloud/service_accounts_controller.rb
View file

@ -45,41 +45,4 @@ class Projects::GoogleCloud::ServiceAccountsController < Projects::GoogleCloud::
rescue Google::Apis::ClientError, Google::Apis::ServerError, Google::Apis::AuthorizationError => error
handle_gcp_error(error, project)
end
private
def validate_gcp_token!
is_token_valid = GoogleApi::CloudPlatform::Client.new(token_in_session, nil)
.validate_token(expires_at_in_session)
return if is_token_valid
return_url = project_google_cloud_index_path(project)
state = generate_session_key_redirect(request.url, return_url)
@authorize_url = GoogleApi::CloudPlatform::Client.new(nil,
callback_google_api_auth_url,
state: state).authorize_url
redirect_to @authorize_url
end
def generate_session_key_redirect(uri, error_uri)
GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key|
session[key] = uri
session[:error_uri] = error_uri
end
end
def token_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_token]
end
def expires_at_in_session
session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at]
end
def handle_gcp_error(error, project)
Gitlab::ErrorTracking.track_exception(error, project_id: project.id)
@js_data = { screen: 'gcp_error', error: error.to_s }.to_json
render status: :unauthorized, template: 'projects/google_cloud/errors/gcp_error'
end
end

1
app/controllers/projects/tree_controller.rb
View file

@ -19,6 +19,7 @@ class Projects::TreeController < Projects::ApplicationController
push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml)
push_frontend_feature_flag(:new_dir_modal, @project, default_enabled: :yaml)
push_frontend_feature_flag(:refactor_blob_viewer, @project, default_enabled: :yaml)
push_frontend_feature_flag(:highlight_js, @project, default_enabled: :yaml)
end
feature_category :source_code_management

1
app/graphql/types/ci/runner_type.rb
View file

@ -4,6 +4,7 @@ module Types
module Ci
class RunnerType < BaseObject
edge_type_class(RunnerWebUrlEdge)
connection_type_class(Types::CountableConnectionType)
graphql_name 'CiRunner'
authorize :read_runner
present_using ::Ci::RunnerPresenter

4
app/models/ci/build.rb
View file

@ -427,6 +427,10 @@ module Ci
action? && !archived? && (manual? || scheduled? || retryable?)
end
def waiting_for_deployment_approval?
manual? && starts_environment? && deployment&.blocked?
end
def schedulable?
self.when == 'delayed' && options[:start_in].present?
end

4
app/policies/group_policy.rb
View file

@ -123,8 +123,6 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
enable :read_group_member
enable :read_custom_emoji
enable :read_counts
enable :read_crm_organization
enable :read_crm_contact
end
rule { ~public_group & ~has_access }.prevent :read_counts
@ -159,6 +157,8 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy
enable :read_prometheus
enable :read_package
enable :read_package_settings
enable :read_crm_organization
enable :read_crm_contact
end
rule { maintainer }.policy do

8
app/services/ci/process_sync_events_service.rb
View file

@ -28,18 +28,16 @@ module Ci
return if events.empty?
first = events.first
last_processed = nil
processed_events = []
begin
events.each do |event|
@sync_class.sync!(event)
last_processed = event
processed_events << event
end
ensure
# remove events till the one that was last succesfully processed
@sync_event_class.id_in(first.id..last_processed.id).delete_all if last_processed
@sync_event_class.id_in(processed_events).delete_all
end
end

1
app/views/admin/application_settings/general.html.haml
View file

@ -118,3 +118,4 @@
= render 'admin/application_settings/snowplow'
= render 'admin/application_settings/eks'
= render 'admin/application_settings/floc'
= render_if_exists 'admin/application_settings/license_file'

4
config/environments/production.rb
View file

@ -36,7 +36,9 @@ Rails.application.configure do
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
# config.force_ssl = true
# See everything in the log (default is :info)
# Include generic and useful information about system operation, but avoid logging too much
# information to avoid inadvertent exposure of personally identifiable information (PII).
# Note: This configuration does not affect the log level of `Gitlab::Logger` and its subclasses.
config.log_level = :info
# Suppress 'Rendered template ...' messages in the log

8
config/feature_flags/development/loose_index_scan_for_distinct_values.yml
View file

@ -1,8 +0,0 @@
---
name: loose_index_scan_for_distinct_values
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55985
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/324210
milestone: '13.10'
type: development
group: group::optimize
default_enabled: false

31
config/initializers/wikicloth_disable_lua_patch.rb Normal file
View file

@ -0,0 +1,31 @@
# frozen_string_literal: true
require 'wikicloth'
require 'wikicloth/extensions/lua'
# Adds patch to disable lua support to eliminate vulnerability to injection attack.
#
# The maintainers are not releasing new versions, so we need to patch it here.
#
# If they ever do release a version which contains a fix for this, then we can remove this file.
#
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/345892#note_751107320
# Guard to ensure we remember to delete this patch if they ever release a new version of wikicloth
# which disables Lua by default or otherwise eliminates all vulnerabilities mentioned in
# https://gitlab.com/gitlab-org/gitlab/-/issues/345892, including the possibility of an HTML/JS
# injection attack as mentioned in https://gitlab.com/gitlab-org/gitlab/-/issues/345892#note_751981608
unless Gem::Version.new(WikiCloth::VERSION) == Gem::Version.new('0.8.1')
raise 'New version of WikiCloth detected, please either update the version for this check, ' \
'or remove this patch if no longer needed'
end
module WikiCloth
class LuaExtension < Extension
protected
def init_lua
@options[:disable_lua] = true
end
end
end

0
config/initializers/wikicloth_patch.rb → config/initializers/wikicloth_redos_patch.rb
View file

3
config/routes/project.rb
View file

@ -319,6 +319,9 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
namespace :google_cloud do
resources :service_accounts, only: [:index, :create]
get '/deployments/cloud_run', to: 'deployments#cloud_run'
get '/deployments/cloud_storage', to: 'deployments#cloud_storage'
end
resources :environments, except: [:destroy] do

15
db/migrate/20220104174445_add_ci_runners_index_on_active_state.rb Normal file
View file

@ -0,0 +1,15 @@
# frozen_string_literal: true
class AddCiRunnersIndexOnActiveState < Gitlab::Database::Migration[1.0]
disable_ddl_transaction!
INDEX_NAME = 'index_ci_runners_on_active'
def up
add_concurrent_index :ci_runners, [:active, :id], name: INDEX_NAME
end
def down
remove_concurrent_index_by_name :ci_runners, INDEX_NAME
end
end

1
db/schema_migrations/20220104174445 Normal file
View file

@ -0,0 +1 @@
5e5e41ee4c8dc9c3fe791470862d15b8d213fcc931ef8b80937bdb5f5db20aed

2
db/structure.sql
View file

@ -25593,6 +25593,8 @@ CREATE INDEX index_ci_runner_projects_on_project_id ON ci_runner_projects USING
CREATE INDEX index_ci_runner_projects_on_runner_id ON ci_runner_projects USING btree (runner_id);
CREATE INDEX index_ci_runners_on_active ON ci_runners USING btree (active, id);
CREATE INDEX index_ci_runners_on_contacted_at_and_id_desc ON ci_runners USING btree (contacted_at, id DESC);
CREATE INDEX index_ci_runners_on_contacted_at_and_id_where_inactive ON ci_runners USING btree (contacted_at DESC, id DESC) WHERE (active = false);

48
doc/administration/logs.md
View file

@ -20,6 +20,54 @@ including adjusting log retention, log forwarding,
switching logs from JSON to plain text logging, and more.
- [How to parse and analyze JSON logs](troubleshooting/log_parsing.md).
## Log Levels
Each log message has an assigned log level that indicates its importance and verbosity.
Each logger has an assigned minimum log level.
A logger emits a log message only if its log level is equal to or above the minimum log level.
The following log levels are supported:
| Level | Name |
|-------|---------|
| 0 | DEBUG |
| 1 | INFO |
| 2 | WARN |
| 3 | ERROR |
| 4 | FATAL |
| 5 | UNKNOWN |
GitLab loggers emit all log messages because they are set to `DEBUG` by default.
### Override default log level
You can override the minimum log level for GitLab loggers using the `GITLAB_LOG_LEVEL` environment variable.
Valid values are either a value of `0` to `5`, or the name of the log level.
Example:
```shell
GITLAB_LOG_LEVEL=info
```
For some services, other log levels are in place that are not affected by this setting.
Some of these services have their own environment variables to override the log level. For example:
| Service | Log Level | Environment variable |
|----------------------|-----------|----------------------|
| GitLab API | `INFO` | |
| GitLab Cleanup | `INFO` | `DEBUG` |
| GitLab Doctor | `INFO` | `VERBOSE` |
| GitLab Export | `INFO` | `EXPORT_DEBUG` |
| GitLab Geo | `INFO` | |
| GitLab Import | `INFO` | `IMPORT_DEBUG` |
| GitLab QA Runtime | `ERROR` | `QA_DEBUG` |
| Google APIs | `INFO` | |
| Rack Timeout | `ERROR` | |
| Sidekiq (server) | `INFO` | |
| Snowplow Tracker | `FATAL` | |
| gRPC Client (Gitaly) | `WARN` | `GRPC_LOG_LEVEL` |
## Log Rotation
The logs for a given service may be managed and rotated by:

1
doc/api/graphql/reference/index.md
View file

@ -5503,6 +5503,7 @@ The connection type for [`CiRunner`](#cirunner).
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="cirunnerconnectioncount"></a>`count` | [`Int!`](#int) | Total count of collection. |
| <a id="cirunnerconnectionedges"></a>`edges` | [`[CiRunnerEdge]`](#cirunneredge) | A list of edges. |
| <a id="cirunnerconnectionnodes"></a>`nodes` | [`[CiRunner]`](#cirunner) | A list of nodes. |
| <a id="cirunnerconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. |

81
doc/development/database/efficient_in_operator_queries.md
View file

@ -589,6 +589,87 @@ LIMIT 20
NOTE:
To make the query efficient, the following columns need to be covered with an index: `project_id`, `issue_type`, `created_at`, and `id`.
#### Using calculated ORDER BY expression
The following example orders epic records by the duration between the creation time and closed
time. It is calculated with the following formula:
```sql
SELECT EXTRACT('epoch' FROM epics.closed_at - epics.created_at) FROM epics
```
The query above returns the duration in seconds (`double precision`) between the two timestamp
columns in seconds. To order the records by this expression, you must reference it
in the `ORDER BY` clause:
```sql
SELECT EXTRACT('epoch' FROM epics.closed_at - epics.created_at)
FROM epics
ORDER BY EXTRACT('epoch' FROM epics.closed_at - epics.created_at) DESC
```
To make this ordering efficient on the group-level with the in-operator optimization, use a
custom `ORDER BY` configuration. Since the duration is not a distinct value (no unique index
present), you must add a tie-breaker column (`id`).
The following example shows the final `ORDER BY` clause:
```sql
ORDER BY extract('epoch' FROM epics.closed_at - epics.created_at) DESC, epics.id DESC
```
Snippet for loading records ordered by the calcualted duration:
```ruby
arel_table = Epic.arel_table
order = Gitlab::Pagination::Keyset::Order.build([
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: 'duration_in_seconds',
order_expression: Arel.sql('EXTRACT(EPOCH FROM epics.closed_at - epics.created_at)').desc,
distinct: false,
sql_type: 'double precision' # important for calculated SQL expressions
),
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: 'id',
order_expression: arel_table[:id].desc
)
])
records = Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new(
scope: Epic.where.not(closed_at: nil).reorder(order), # filter out NULL values
array_scope: Group.find(9970).self_and_descendants.select(:id),
array_mapping_scope: -> (id_expression) { Epic.where(Epic.arel_table[:group_id].eq(id_expression)) }
).execute.limit(20)
puts records.pluck(:duration_in_seconds, :id) # other columnns are not available
```
Building the query requires quite a bit of configuration. For the order configuration you
can find more information within the
[complex order configuration](keyset_pagination.md#complex-order-configuration)
section for keyset paginated database queries.
The query requires a specialized database index:
```sql
CREATE INDEX index_epics_on_duration ON epics USING btree (group_id, EXTRACT(EPOCH FROM epics.closed_at - epics.created_at) DESC, id DESC) WHERE (closed_at IS NOT NULL);
```
Notice that the `finder_query` parameter is not used. The query only returns the `ORDER BY` columns
which are the `duration_in_seconds` (calculated column) and the `id` columns. This is a limitation
of the feature, defining the `finder_query` with calculated `ORDER BY` expressions is not supported.
To get the complete database records, an extra query can be invoked by the returned `id` column:
```ruby
records_by_id = records.index_by(&:id)
complete_records = Epic.where(id: records_by_id.keys).index_by(&:id)
# Printing the complete records according to the `ORDER BY` clause
records_by_id.each do |id, _|
puts complete_records[id].attributes
end
```
#### Batch iteration
Batch iteration over the records is possible via the keyset `Iterator` class.

7
doc/user/admin_area/license.md
View file

@ -50,13 +50,14 @@ If you receive a license file from GitLab (for example a new trial), you can upl
The first time you visit your GitLab EE installation signed in as an administrator,
you should see a note urging you to upload a license with a link that takes you
to the **Subscription** area.
to the **Upload license** page.
Otherwise, to manually go to the **Subscription** area:
Otherwise, to manually go to the **Upload license** page:
1. Sign in to your GitLab self-managed instance.
1. From the top menu, select the Admin Area **{admin}**.
1. From the left sidebar, select **Subscription**, and select **Upload a license file**.
1. On the left sidebar, select **Settings**.
1. In the **License file** section, select **Upload a license**.
- *If you've received a `.gitlab-license` file:*
1. Download the license file to your local machine.

BIN
doc/user/analytics/img/repository_analytics_v13_0.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 34 KiB

33
doc/user/analytics/repository_analytics.md
View file

@ -6,36 +6,33 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Repository Analytics **(FREE)**
Get high-level overview of the project's Git repository.
Use Repository Analytics to view information about a project's Git repository:
![Repository Analytics](img/repository_analytics_v13_0.png)
## Availability
- Programming languages used in the repository.
- Code coverage history from last 3 months ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1).
- Commit statistics (last month).
- Commits per day of month.
- Commits per weekday.
- Commits per day hour (UTC).
Repository Analytics is part of [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss). It's available to anyone who has permission to clone the repository.
The feature requires:
Repository Analytics requires:
- An initialized Git repository.
- At least one commit in the default branch (`master` by default).
## Overview
You can find Repository Analytics in the project's sidebar. To access the page, go to **{chart}** **Analytics > Repository**.
NOTE:
Without a Git commit in the default branch, the menu item won't be visible.
Commits in a project's [wiki](../project/wiki/index.md#track-wiki-events) are not included in the analysis.
### Charts
## View Repository Analytics
The data in the charts are queued. Background workers update the charts 10 minutes after each commit in the default branch. Depending on the size of the GitLab installation, it may take longer for data to refresh due to variations in the size of background job queues.
To review Repository Analytics for a project:
Available charts:
1. On the top bar, select **Menu > Projects** and find your project.
1. On the left sidebar, select **Analytics > Repository**.
- Programming languages used in the repository
- Code coverage history (last 3 months) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1)
- Commit statistics (last month)
- Commits per day of month
- Commits per weekday
- Commits per day hour (UTC)
## How Repository Analytics chart data is updated
Data in the charts are queued. Background workers update the charts 10 minutes after each commit in the default branch. Depending on the size of the GitLab installation, it may take longer for data to refresh due to variations in the size of background job queues.

13
doc/user/crm/index.md
View file

@ -14,6 +14,15 @@ With customer relations management (CRM) you can create a record of contacts
You can use contacts and organizations to tie work to customers for billing and reporting purposes.
To read more about what is planned for the future, see [issue 2256](https://gitlab.com/gitlab-org/gitlab/-/issues/2256).
## Permissions
| Permission | Guest | Reporter | Developer, Maintainer, and Owner |
| ---------- | ---------------- | -------- | -------------------------------- |
| View contacts/organizations | | ✓ | ✓ |
| View issue contacts | | ✓ | ✓ |
| Add/remove issue contacts | | ✓ | ✓ |
| Create/edit contacts/organizations | | | ✓ |
## Enable customer relations management (CRM)
To enable customer relations management in a group:
@ -122,10 +131,6 @@ API.
### Add or remove issue contacts
Prerequisites:
- You must have at least the [Developer role](../permissions.md#project-members-permissions) for a group.
### Add contacts to an issue
To add contacts to an issue use the `/add_contacts`

3
lib/gitlab/ci/status/build/factory.rb
View file

@ -14,7 +14,8 @@ module Gitlab
Status::Build::WaitingForResource,
Status::Build::Preparing,
Status::Build::Pending,
Status::Build::Skipped],
Status::Build::Skipped,
Status::Build::WaitingForApproval],
[Status::Build::Cancelable,
Status::Build::Retryable],
[Status::Build::FailedUnmetPrerequisites,

24
lib/gitlab/ci/status/build/waiting_for_approval.rb Normal file
View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
module Gitlab
module Ci
module Status
module Build
class WaitingForApproval < Status::Extended
def illustration
{
image: 'illustrations/manual_action.svg',
size: 'svg-394',
title: 'Waiting for approval',
content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment."
}
end
def self.matches?(build, user)
build.waiting_for_deployment_approval?
end
end
end
end
end
end

29
lib/gitlab/database/batch_counter.rb
View file

@ -52,12 +52,7 @@ module Gitlab
batch_end = [batch_start + batch_size, finish].min
batch_relation = build_relation_batch(batch_start, batch_end, mode)
op_args = @operation_args
if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode)
op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS]
end
results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend
results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend
batch_start = batch_end
rescue ActiveRecord::QueryCanceled => error
# retry with a safe batch size & warmer cache
@ -67,18 +62,6 @@ module Gitlab
log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error)
return FALLBACK
end
rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error
Gitlab::AppJsonLogger
.error(
event: 'batch_count',
relation: @relation.table_name,
operation: @operation,
operation_args: @operation_args,
mode: mode,
message: "LooseIndexScanDistinctCount column error: #{error.message}"
)
return FALLBACK
end
sleep(SLEEP_TIME_IN_SECONDS)
@ -104,11 +87,7 @@ module Gitlab
private
def build_relation_batch(start, finish, mode)
if use_loose_index_scan_for_distinct_values?(mode)
Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish)
else
@relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
end
@relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
end
def batch_size_for_mode_and_operation(mode, operation)
@ -151,10 +130,6 @@ module Gitlab
)
end
def use_loose_index_scan_for_distinct_values?(mode)
Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct
end
def not_group_by_query?
!@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank?
end

102
lib/gitlab/database/loose_index_scan_distinct_count.rb
View file

@ -1,102 +0,0 @@
# frozen_string_literal: true
module Gitlab
module Database
# This class builds efficient batched distinct query by using loose index scan.
# Consider the following example:
# > Issue.distinct(:project_id).where(project_id: (1...100)).count
#
# Note: there is an index on project_id
#
# This query will read each element in the index matching the project_id filter.
# If for a project_id has 100_000 issues, all 100_000 elements will be read.
#
# A loose index scan will only read one entry from the index for each project_id to reduce the number of disk reads.
#
# Usage:
#
# Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100)
#
# The query will return the number of distinct projects_ids between 1 and 100
#
# Getting the Arel query:
#
# Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100)
class LooseIndexScanDistinctCount
COLUMN_ALIAS = 'distinct_count_column'
ColumnConfigurationError = Class.new(StandardError)
def initialize(scope, column)
if scope.is_a?(ActiveRecord::Relation)
@scope = scope
@model = scope.model
else
@scope = scope.where({})
@model = scope
end
@column = transform_column(column)
end
def count(from:, to:)
build_query(from: from, to: to).count(COLUMN_ALIAS)
end
def build_query(from:, to:) # rubocop:disable Metrics/AbcSize
cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false })
table = model.arel_table
cte << @scope
.dup
.select(column.as(COLUMN_ALIAS))
.where(column.gteq(from))
.where(column.lt(to))
.order(column)
.limit(1)
inner_query = @scope
.dup
.where(column.gt(cte.table[COLUMN_ALIAS]))
.where(column.lt(to))
.select(column.as(COLUMN_ALIAS))
.order(column)
.limit(1)
cte << cte.table
.project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS))
.where(cte.table[COLUMN_ALIAS].lt(to))
model
.with
.recursive(cte.to_arel)
.from(cte.alias_to(table))
.unscope(where: :source_type)
.unscope(where: model.inheritance_column) # Remove STI query, not needed here
end
private
attr_reader :column, :model
# Transforms the column so it can be used in Arel expressions
#
# 'table.column' => 'table.column'
# 'column' => 'table_name.column'
# :column => 'table_name.column'
# Arel::Attributes::Attribute => name of the column
def transform_column(column)
if column.is_a?(String) || column.is_a?(Symbol)
column_as_string = column.to_s
column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.')
Arel.sql(column_as_string)
elsif column.is_a?(Arel::Attributes::Attribute)
column
else
raise ColumnConfigurationError, "Cannot transform the column: #{column.inspect}, please provide the column name as string"
end
end
end
end
end

6
lib/gitlab/logger.rb
View file

@ -33,7 +33,11 @@ module Gitlab
def self.build
Gitlab::SafeRequestStore[self.cache_key] ||=
new(self.full_log_path, level: ::Logger::DEBUG)
new(self.full_log_path, level: log_level)
end
def self.log_level(fallback: ::Logger::DEBUG)
ENV.fetch('GITLAB_LOG_LEVEL', fallback)
end
def self.full_log_path

25
lib/gitlab/pagination/keyset/column_order_definition.rb
View file

@ -114,6 +114,20 @@ module Gitlab
# - When the order is a calculated expression or the column is in another table (JOIN-ed)
#
# If the add_to_projections is true, the query builder will automatically add the column to the SELECT values
#
# **sql_type**
#
# The SQL type of the column or SQL expression. This is an optional field which is only required when using the
# column with the InOperatorOptimization class.
#
# Example: When the order expression is a calculated SQL expression.
#
# {
# attribute_name: 'id_times_count',
# order_expression: Arel.sql('(id * count)').asc,
# sql_type: 'integer' # the SQL type here must match with the type of the produced data by the order_expression. Putting 'text' here would be incorrect.
# }
#
class ColumnOrderDefinition
REVERSED_ORDER_DIRECTIONS = { asc: :desc, desc: :asc }.freeze
REVERSED_NULL_POSITIONS = { nulls_first: :nulls_last, nulls_last: :nulls_first }.freeze
@ -122,7 +136,8 @@ module Gitlab
attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections, :order_direction
def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false)
# rubocop: disable Metrics/ParameterLists
def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, sql_type: nil, add_to_projections: false)
@attribute_name = attribute_name
@order_expression = order_expression
@column_expression = column_expression || calculate_column_expression(order_expression)
@ -130,8 +145,10 @@ module Gitlab
@reversed_order_expression = reversed_order_expression || calculate_reversed_order(order_expression)
@nullable = parse_nullable(nullable, distinct)
@order_direction = parse_order_direction(order_expression, order_direction)
@sql_type = sql_type
@add_to_projections = add_to_projections
end
# rubocop: enable Metrics/ParameterLists
def reverse
self.class.new(
@ -185,6 +202,12 @@ module Gitlab
sql_string
end
def sql_type
raise Gitlab::Pagination::Keyset::SqlTypeMissingError.for_column(self) if @sql_type.nil?
@sql_type
end
private
attr_reader :reversed_order_expression, :nullable, :distinct

19
lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb
View file

@ -4,23 +4,35 @@ module Gitlab
module Pagination
module Keyset
module InOperatorOptimization
# This class is used for wrapping an Arel column with
# convenient helper methods in order to make the query
# building for the InOperatorOptimization a bit cleaner.
class ColumnData
attr_reader :original_column_name, :as, :arel_table
def initialize(original_column_name, as, arel_table)
@original_column_name = original_column_name.to_s
# column - name of the DB column
# as - custom alias for the column
# arel_table - relation where the column is located
def initialize(column, as, arel_table)
@original_column_name = column
@as = as.to_s
@arel_table = arel_table
end
# Generates: `issues.name AS my_alias`
def projection
arel_column.as(as)
end
# Generates: issues.name`
def arel_column
arel_table[original_column_name]
end
# overridden in OrderByColumnData class
alias_method :column_expression, :arel_column
# Generates: `issues.my_alias`
def arel_column_as
arel_table[as]
end
@ -29,8 +41,9 @@ module Gitlab
"#{arel_table.name}_#{original_column_name}_array"
end
# Generates: SELECT ARRAY_AGG(...) AS issues_name_array
def array_aggregated_column
Arel::Nodes::NamedFunction.new('ARRAY_AGG', [arel_column]).as(array_aggregated_column_name)
Arel::Nodes::NamedFunction.new('ARRAY_AGG', [column_expression]).as(array_aggregated_column_name)
end
end
end

37
lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data.rb Normal file
View file

@ -0,0 +1,37 @@
# frozen_string_literal: true
module Gitlab
module Pagination
module Keyset
module InOperatorOptimization
class OrderByColumnData < ColumnData
extend ::Gitlab::Utils::Override
attr_reader :column
# column - a ColumnOrderDefinition object
# as - custom alias for the column
# arel_table - relation where the column is located
def initialize(column, as, arel_table)
super(column.attribute_name.to_s, as, arel_table)
@column = column
end
override :arel_column
def arel_column
column.column_expression
end
override :column_expression
def column_expression
arel_table[original_column_name]
end
def column_for_projection
column.column_expression.as(original_column_name)
end
end
end
end
end
end

6
lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb
View file

@ -9,16 +9,16 @@ module Gitlab
# This class exposes collection methods for the order by columns
#
# Example: by modelling the `issues.created_at ASC, issues.id ASC` ORDER BY
# Example: by modeling the `issues.created_at ASC, issues.id ASC` ORDER BY
# SQL clause, this class will receive two ColumnOrderDefinition objects
def initialize(columns, arel_table)
@columns = columns.map do |column|
ColumnData.new(column.attribute_name, "order_by_columns_#{column.attribute_name}", arel_table)
OrderByColumnData.new(column, "order_by_columns_#{column.attribute_name}", arel_table)
end
end
def arel_columns
columns.map(&:arel_column)
columns.map(&:column_for_projection)
end
def array_aggregated_columns

4
lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb
View file

@ -120,7 +120,7 @@ module Gitlab
.from(array_cte)
.join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE"))
order_by_columns.each { |column| q.where(column.arel_column.not_eq(nil)) }
order_by_columns.each { |column| q.where(column.column_expression.not_eq(nil)) }
q.as('array_scope_lateral_query')
end
@ -231,7 +231,7 @@ module Gitlab
order
.apply_cursor_conditions(keyset_scope, cursor_values, use_union_optimization: true)
.reselect(*order_by_columns.arel_columns)
.reselect(*order_by_columns.map(&:column_for_projection))
.limit(1)
end

15
lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb
View file

@ -12,11 +12,7 @@ module Gitlab
end
def initializer_columns
order_by_columns.map do |column|
column_name = column.original_column_name.to_s
type = model.columns_hash[column_name].sql_type
"NULL::#{type} AS #{column_name}"
end
order_by_columns.map { |column_data| null_with_type_cast(column_data) }
end
def columns
@ -30,6 +26,15 @@ module Gitlab
private
attr_reader :model, :order_by_columns
def null_with_type_cast(column_data)
column_name = column_data.original_column_name.to_s
active_record_column = model.columns_hash[column_name]
type = active_record_column ? active_record_column.sql_type : column_data.column.sql_type
"NULL::#{type} AS #{column_name}"
end
end
end
end

16
lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb
View file

@ -9,6 +9,8 @@ module Gitlab
RECORDS_COLUMN = 'records'
def initialize(finder_query, model, order_by_columns)
verify_order_by_attributes_on_model!(model, order_by_columns)
@finder_query = finder_query
@order_by_columns = order_by_columns
@table_name = model.table_name
@ -34,6 +36,20 @@ module Gitlab
private
attr_reader :finder_query, :order_by_columns, :table_name
def verify_order_by_attributes_on_model!(model, order_by_columns)
order_by_columns.map(&:column).each do |column|
unless model.columns_hash[column.attribute_name.to_s]
text = <<~TEXT
The "RecordLoaderStrategy" does not support the following ORDER BY column because
it's not available on the \"#{model.table_name}\" table: #{column.attribute_name}
Omit the "finder_query" parameter to use the "OrderValuesLoaderStrategy".
TEXT
raise text
end
end
end
end
end
end

19
lib/gitlab/pagination/keyset/sql_type_missing_error.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Gitlab
module Pagination
module Keyset
class SqlTypeMissingError < StandardError
def self.for_column(column)
message = <<~TEXT
The "sql_type" attribute is not set for the following column definition:
#{column.attribute_name}
See the ColumnOrderDefinition class for more context.
TEXT
new(message)
end
end
end
end
end

2
lib/sidebars/projects/menus/infrastructure_menu.rb
View file

@ -100,7 +100,7 @@ module Sidebars
::Sidebars::MenuItem.new(
title: _('Google Cloud'),
link: project_google_cloud_index_path(context.project),
active_routes: { controller: [:google_cloud, :service_accounts] },
active_routes: { controller: [:google_cloud, :service_accounts, :deployments] },
item_id: :google_cloud
)
end

15
locale/gitlab.pot
View file

@ -731,6 +731,9 @@ msgstr ""
msgid "%{link_start}Start the title with %{draft_snippet}%{link_end} to prevent a merge request draft from merging before it's ready."
msgstr ""
msgid "%{link_start}Upload a license%{link_end} file or enter the license key you have received from GitLab Inc."
msgstr ""
msgid "%{link_start}What information does GitLab Inc. collect?%{link_end}"
msgstr ""
@ -6906,6 +6909,9 @@ msgstr ""
msgid "Checkout|(x%{quantity})"
msgstr ""
msgid "Checkout|An unknown error has occurred. Please try again by refreshing this page."
msgstr ""
msgid "Checkout|Billing address"
msgstr ""
@ -39578,6 +39584,9 @@ msgstr ""
msgid "Vulnerability|Information related how the vulnerability was discovered and its impact to the system."
msgstr ""
msgid "Vulnerability|Learn more about this vulnerability and the best way to resolve it."
msgstr ""
msgid "Vulnerability|Links"
msgstr ""
@ -39626,6 +39635,12 @@ msgstr ""
msgid "Vulnerability|Tool"
msgstr ""
msgid "Vulnerability|Training"
msgstr ""
msgid "Vulnerability|Training not available for this vulnerability."
msgstr ""
msgid "Vulnerability|Unmodified Response"
msgstr ""

49
spec/lib/gitlab/ci/status/build/waiting_for_approval_spec.rb Normal file
View file

@ -0,0 +1,49 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Status::Build::WaitingForApproval do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { create(:user) }
subject { described_class.new(Gitlab::Ci::Status::Core.new(build, user)) }
describe '#illustration' do
let(:build) { create(:ci_build, :manual, environment: 'production', project: project) }
before do
environment = create(:environment, name: 'production', project: project)
create(:deployment, :blocked, project: project, environment: environment, deployable: build)
end
it { expect(subject.illustration).to include(:image, :size) }
it { expect(subject.illustration[:title]).to eq('Waiting for approval') }
it { expect(subject.illustration[:content]).to include('This job deploys to the protected environment "production"') }
end
describe '.matches?' do
subject { described_class.matches?(build, user) }
let(:build) { create(:ci_build, :manual, environment: 'production', project: project) }
before do
create(:deployment, deployment_status, deployable: build, project: project)
end
context 'when build is waiting for approval' do
let(:deployment_status) { :blocked }
it 'is a correct match' do
expect(subject).to be_truthy
end
end
context 'when build is not waiting for approval' do
let(:deployment_status) { :created }
it 'does not match' do
expect(subject).to be_falsey
end
end
end
end

76
spec/lib/gitlab/database/batch_count_spec.rb
View file

@ -270,8 +270,6 @@ RSpec.describe Gitlab::Database::BatchCount do
end
it "defaults the batch size to #{Gitlab::Database::BatchCounter::DEFAULT_DISTINCT_BATCH_SIZE}" do
stub_feature_flags(loose_index_scan_for_distinct_values: false)
min_id = model.minimum(:id)
relation = instance_double(ActiveRecord::Relation)
allow(model).to receive_message_chain(:select, public_send: relation)
@ -317,85 +315,13 @@ RSpec.describe Gitlab::Database::BatchCount do
end
end
context 'when the loose_index_scan_for_distinct_values feature flag is off' do
it_behaves_like 'when batch fetch query is canceled' do
let(:mode) { :distinct }
let(:operation) { :count }
let(:operation_args) { nil }
let(:column) { nil }
subject { described_class.method(:batch_distinct_count) }
before do
stub_feature_flags(loose_index_scan_for_distinct_values: false)
end
end
end
context 'when the loose_index_scan_for_distinct_values feature flag is on' do
it_behaves_like 'when batch fetch query is canceled' do
let(:mode) { :distinct }
let(:operation) { :count }
let(:operation_args) { nil }
let(:column) { nil }
let(:batch_size) { 10_000 }
subject { described_class.method(:batch_distinct_count) }
before do
stub_feature_flags(loose_index_scan_for_distinct_values: true)
end
it 'reduces batch size by half and retry fetch' do
too_big_batch_relation_mock = instance_double(ActiveRecord::Relation)
count_method = double(send: 1)
allow(too_big_batch_relation_mock).to receive(:send).and_raise(ActiveRecord::QueryCanceled)
allow(Gitlab::Database::LooseIndexScanDistinctCount).to receive_message_chain(:new, :build_query).with(from: 0, to: batch_size).and_return(too_big_batch_relation_mock)
allow(Gitlab::Database::LooseIndexScanDistinctCount).to receive_message_chain(:new, :build_query).with(from: 0, to: batch_size / 2).and_return(count_method)
allow(Gitlab::Database::LooseIndexScanDistinctCount).to receive_message_chain(:new, :build_query).with(from: batch_size / 2, to: batch_size).and_return(count_method)
subject.call(model, column, batch_size: batch_size, start: 0, finish: batch_size - 1)
end
context 'when all retries fail' do
let(:batch_count_query) { 'SELECT COUNT(id) FROM relation WHERE id BETWEEN 0 and 1' }
before do
relation = instance_double(ActiveRecord::Relation)
allow(Gitlab::Database::LooseIndexScanDistinctCount).to receive_message_chain(:new, :build_query).and_return(relation)
allow(relation).to receive(:send).and_raise(ActiveRecord::QueryCanceled.new('query timed out'))
allow(relation).to receive(:to_sql).and_return(batch_count_query)
end
it 'logs failing query' do
expect(Gitlab::AppJsonLogger).to receive(:error).with(
event: 'batch_count',
relation: model.table_name,
operation: operation,
operation_args: operation_args,
start: 0,
mode: mode,
query: batch_count_query,
message: 'Query has been canceled with message: query timed out'
)
expect(subject.call(model, column, batch_size: batch_size, start: 0)).to eq(-1)
end
end
context 'when LooseIndexScanDistinctCount raises error' do
let(:column) { :creator_id }
let(:error_class) { Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError }
it 'rescues ColumnConfigurationError' do
allow(Gitlab::Database::LooseIndexScanDistinctCount).to receive(:new).and_raise(error_class.new('error message'))
expect(Gitlab::AppJsonLogger).to receive(:error).with(a_hash_including(message: 'LooseIndexScanDistinctCount column error: error message'))
expect(subject.call(Project, column, batch_size: 10_000, start: 0)).to eq(-1)
end
end
end
end

71
spec/lib/gitlab/database/loose_index_scan_distinct_count_spec.rb
View file

@ -1,71 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::LooseIndexScanDistinctCount do
context 'counting distinct users' do
let_it_be(:user) { create(:user) }
let_it_be(:other_user) { create(:user) }
let(:column) { :creator_id }
before_all do
create_list(:project, 3, creator: user)
create_list(:project, 1, creator: other_user)
end
subject(:count) { described_class.new(Project, :creator_id).count(from: Project.minimum(:creator_id), to: Project.maximum(:creator_id) + 1) }
it { is_expected.to eq(2) }
context 'when STI model is queried' do
it 'does not raise error' do
expect { described_class.new(Group, :owner_id).count(from: 0, to: 1) }.not_to raise_error
end
end
context 'when model with default_scope is queried' do
it 'does not raise error' do
expect { described_class.new(GroupMember, :id).count(from: 0, to: 1) }.not_to raise_error
end
end
context 'when the fully qualified column is given' do
let(:column) { 'projects.creator_id' }
it { is_expected.to eq(2) }
end
context 'when AR attribute is given' do
let(:column) { Project.arel_table[:creator_id] }
it { is_expected.to eq(2) }
end
context 'when invalid value is given for the column' do
let(:column) { Class.new }
it { expect { described_class.new(Group, column) }.to raise_error(Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError) }
end
context 'when null values are present' do
before do
create_list(:project, 2).each { |p| p.update_column(:creator_id, nil) }
end
it { is_expected.to eq(2) }
end
end
context 'counting STI models' do
let!(:groups) { create_list(:group, 3) }
let!(:namespaces) { create_list(:namespace, 2) }
let(:max_id) { Namespace.maximum(:id) + 1 }
it 'counts groups' do
count = described_class.new(Group, :id).count(from: 0, to: max_id)
expect(count).to eq(3)
end
end
end

94
spec/lib/gitlab/logger_spec.rb Normal file
View file

@ -0,0 +1,94 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Logger do
describe '.build' do
before do
allow(described_class).to receive(:file_name_noext).and_return('log')
end
subject { described_class.build }
it 'builds logger using Gitlab::Logger.log_level' do
expect(described_class).to receive(:log_level).and_return(:warn)
expect(subject.level).to eq(described_class::WARN)
end
it 'raises ArgumentError if invalid log level' do
allow(described_class).to receive(:log_level).and_return(:invalid)
expect { subject.level }.to raise_error(ArgumentError, 'invalid log level: invalid')
end
using RSpec::Parameterized::TableSyntax
where(:env_value, :resulting_level) do
0 | described_class::DEBUG
:debug | described_class::DEBUG
'debug' | described_class::DEBUG
'DEBUG' | described_class::DEBUG
'DeBuG' | described_class::DEBUG
1 | described_class::INFO
:info | described_class::INFO
'info' | described_class::INFO
'INFO' | described_class::INFO
'InFo' | described_class::INFO
2 | described_class::WARN
:warn | described_class::WARN
'warn' | described_class::WARN
'WARN' | described_class::WARN
'WaRn' | described_class::WARN
3 | described_class::ERROR
:error | described_class::ERROR
'error' | described_class::ERROR
'ERROR' | described_class::ERROR
'ErRoR' | described_class::ERROR
4 | described_class::FATAL
:fatal | described_class::FATAL
'fatal' | described_class::FATAL
'FATAL' | described_class::FATAL
'FaTaL' | described_class::FATAL
5 | described_class::UNKNOWN
:unknown | described_class::UNKNOWN
'unknown' | described_class::UNKNOWN
'UNKNOWN' | described_class::UNKNOWN
'UnKnOwN' | described_class::UNKNOWN
end
with_them do
it 'builds logger if valid log level' do
stub_env('GITLAB_LOG_LEVEL', env_value)
expect(subject.level).to eq(resulting_level)
end
end
end
describe '.log_level' do
context 'if GITLAB_LOG_LEVEL is set' do
before do
stub_env('GITLAB_LOG_LEVEL', described_class::ERROR)
end
it 'returns value of GITLAB_LOG_LEVEL' do
expect(described_class.log_level).to eq(described_class::ERROR)
end
it 'ignores fallback' do
expect(described_class.log_level(fallback: described_class::FATAL)).to eq(described_class::ERROR)
end
end
context 'if GITLAB_LOG_LEVEL is not set' do
it 'returns default fallback DEBUG' do
expect(described_class.log_level).to eq(described_class::DEBUG)
end
it 'returns passed fallback' do
expect(described_class.log_level(fallback: described_class::FATAL)).to eq(described_class::FATAL)
end
end
end
end

35
spec/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_column_data_spec.rb Normal file
View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::OrderByColumnData do
let(:arel_table) { Issue.arel_table }
let(:column) do
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: :id,
column_expression: arel_table[:id],
order_expression: arel_table[:id].desc
)
end
subject(:column_data) { described_class.new(column, 'column_alias', arel_table) }
describe '#arel_column' do
it 'delegates to column_expression' do
expect(column_data.arel_column).to eq(column.column_expression)
end
end
describe '#column_for_projection' do
it 'returns the expression with AS using the original column name' do
expect(column_data.column_for_projection.to_sql).to eq('"issues"."id" AS id')
end
end
describe '#projection' do
it 'returns the expression with AS using the specified column lias' do
expect(column_data.projection.to_sql).to eq('"issues"."id" AS column_alias')
end
end
end

73
spec/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder_spec.rb
View file

@ -33,14 +33,14 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder
]
end
shared_examples 'correct ordering examples' do
let(:iterator) do
Gitlab::Pagination::Keyset::Iterator.new(
scope: scope.limit(batch_size),
in_operator_optimization_options: in_operator_optimization_options
)
end
let(:iterator) do
Gitlab::Pagination::Keyset::Iterator.new(
scope: scope.limit(batch_size),
in_operator_optimization_options: in_operator_optimization_options
)
end
shared_examples 'correct ordering examples' do |opts = {}|
let(:all_records) do
all_records = []
iterator.each_batch(of: batch_size) do |records|
@ -49,8 +49,10 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder
all_records
end
it 'returns records in correct order' do
expect(all_records).to eq(expected_order)
unless opts[:skip_finder_query_test]
it 'returns records in correct order' do
expect(all_records).to eq(expected_order)
end
end
context 'when not passing the finder query' do
@ -248,4 +250,57 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder
expect { described_class.new(**options).execute }.to raise_error(/The order on the scope does not support keyset pagination/)
end
context 'when ordering by SQL expression' do
let(:order) do
# ORDER BY (id * 10), id
Gitlab::Pagination::Keyset::Order.build([
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: 'id_multiplied_by_ten',
order_expression: Arel.sql('(id * 10)').asc,
sql_type: 'integer'
),
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: :id,
order_expression: Issue.arel_table[:id].asc
)
])
end
let(:scope) { Issue.reorder(order) }
let(:expected_order) { issues.sort_by(&:id) }
let(:in_operator_optimization_options) do
{
array_scope: Project.where(namespace_id: top_level_group.self_and_descendants.select(:id)).select(:id),
array_mapping_scope: -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) }
}
end
context 'when iterating records one by one' do
let(:batch_size) { 1 }
it_behaves_like 'correct ordering examples', skip_finder_query_test: true
end
context 'when iterating records with LIMIT 3' do
let(:batch_size) { 3 }
it_behaves_like 'correct ordering examples', skip_finder_query_test: true
end
context 'when passing finder query' do
let(:batch_size) { 3 }
it 'raises error, loading complete rows are not supported with SQL expressions' do
in_operator_optimization_options[:finder_query] = -> (_, _) { Issue.select(:id, '(id * 10)').where(id: -1) }
expect(in_operator_optimization_options[:finder_query]).not_to receive(:call)
expect do
iterator.each_batch(of: batch_size) { |records| records.to_a }
end.to raise_error /The "RecordLoaderStrategy" does not support/
end
end
end
end

37
spec/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy_spec.rb
View file

@ -31,4 +31,41 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::Strategies::O
])
end
end
context 'when an SQL expression is given' do
context 'when the sql_type attribute is missing' do
let(:order) do
Gitlab::Pagination::Keyset::Order.build([
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: 'id_times_ten',
order_expression: Arel.sql('id * 10').asc
)
])
end
let(:keyset_scope) { Project.order(order) }
it 'raises error' do
expect { strategy.initializer_columns }.to raise_error(Gitlab::Pagination::Keyset::SqlTypeMissingError)
end
end
context 'when the sql_type_attribute is present' do
let(:order) do
Gitlab::Pagination::Keyset::Order.build([
Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
attribute_name: 'id_times_ten',
order_expression: Arel.sql('id * 10').asc,
sql_type: 'integer'
)
])
end
let(:keyset_scope) { Project.order(order) }
it 'returns the initializer columns' do
expect(strategy.initializer_columns).to eq(['NULL::integer AS id_times_ten'])
end
end
end
end

40
spec/policies/group_policy_spec.rb
View file

@ -11,8 +11,6 @@ RSpec.describe GroupPolicy do
it do
expect_allowed(:read_group)
expect_allowed(:read_crm_organization)
expect_allowed(:read_crm_contact)
expect_allowed(:read_counts)
expect_allowed(*read_group_permissions)
expect_disallowed(:upload_file)
@ -21,11 +19,13 @@ RSpec.describe GroupPolicy do
expect_disallowed(*maintainer_permissions)
expect_disallowed(*owner_permissions)
expect_disallowed(:read_namespace)
expect_disallowed(:read_crm_organization)
expect_disallowed(:read_crm_contact)
end
end
context 'with no user and public project' do
let(:project) { create(:project, :public) }
let(:project) { create(:project, :public, group: create(:group, :crm_enabled)) }
let(:current_user) { nil }
before do
@ -41,7 +41,7 @@ RSpec.describe GroupPolicy do
end
context 'with foreign user and public project' do
let(:project) { create(:project, :public) }
let(:project) { create(:project, :public, group: create(:group, :crm_enabled)) }
let(:current_user) { create(:user) }
before do
@ -67,7 +67,7 @@ RSpec.describe GroupPolicy do
it { expect_allowed(*read_group_permissions) }
context 'in subgroups' do
let(:subgroup) { create(:group, :private, parent: group) }
let(:subgroup) { create(:group, :private, :crm_enabled, parent: group) }
let(:project) { create(:project, namespace: subgroup) }
it { expect_allowed(*read_group_permissions) }
@ -235,7 +235,7 @@ RSpec.describe GroupPolicy do
describe 'private nested group use the highest access level from the group and inherited permissions' do
let_it_be(:nested_group) do
create(:group, :private, :owner_subgroup_creation_only, parent: group)
create(:group, :private, :owner_subgroup_creation_only, :crm_enabled, parent: group)
end
before_all do
@ -342,7 +342,7 @@ RSpec.describe GroupPolicy do
let(:current_user) { owner }
context 'when the group share_with_group_lock is enabled' do
let(:group) { create(:group, share_with_group_lock: true, parent: parent) }
let(:group) { create(:group, :crm_enabled, share_with_group_lock: true, parent: parent) }
before do
group.add_owner(owner)
@ -350,10 +350,10 @@ RSpec.describe GroupPolicy do
context 'when the parent group share_with_group_lock is enabled' do
context 'when the group has a grandparent' do
let(:parent) { create(:group, share_with_group_lock: true, parent: grandparent) }
let(:parent) { create(:group, :crm_enabled, share_with_group_lock: true, parent: grandparent) }
context 'when the grandparent share_with_group_lock is enabled' do
let(:grandparent) { create(:group, share_with_group_lock: true) }
let(:grandparent) { create(:group, :crm_enabled, share_with_group_lock: true) }
context 'when the current_user owns the parent' do
before do
@ -379,7 +379,7 @@ RSpec.describe GroupPolicy do
end
context 'when the grandparent share_with_group_lock is disabled' do
let(:grandparent) { create(:group) }
let(:grandparent) { create(:group, :crm_enabled) }
context 'when the current_user owns the parent' do
before do
@ -396,7 +396,7 @@ RSpec.describe GroupPolicy do
end
context 'when the group does not have a grandparent' do
let(:parent) { create(:group, share_with_group_lock: true) }
let(:parent) { create(:group, :crm_enabled, share_with_group_lock: true) }
context 'when the current_user owns the parent' do
before do
@ -413,7 +413,7 @@ RSpec.describe GroupPolicy do
end
context 'when the parent group share_with_group_lock is disabled' do
let(:parent) { create(:group) }
let(:parent) { create(:group, :crm_enabled) }
it { expect_allowed(:change_share_with_group_lock) }
end
@ -698,7 +698,7 @@ RSpec.describe GroupPolicy do
end
it_behaves_like 'clusterable policies' do
let(:clusterable) { create(:group) }
let(:clusterable) { create(:group, :crm_enabled) }
let(:cluster) do
create(:cluster,
:provided_by_gcp,
@ -708,7 +708,7 @@ RSpec.describe GroupPolicy do
end
describe 'update_max_artifacts_size' do
let(:group) { create(:group, :public) }
let(:group) { create(:group, :public, :crm_enabled) }
context 'when no user' do
let(:current_user) { nil }
@ -738,7 +738,7 @@ RSpec.describe GroupPolicy do
end
describe 'design activity' do
let_it_be(:group) { create(:group, :public) }
let_it_be(:group) { create(:group, :public, :crm_enabled) }
let(:current_user) { nil }
@ -935,8 +935,6 @@ RSpec.describe GroupPolicy do
it { is_expected.to be_allowed(:read_package) }
it { is_expected.to be_allowed(:read_group) }
it { is_expected.to be_allowed(:read_crm_organization) }
it { is_expected.to be_allowed(:read_crm_contact) }
it { is_expected.to be_disallowed(:create_package) }
end
@ -946,8 +944,6 @@ RSpec.describe GroupPolicy do
it { is_expected.to be_allowed(:create_package) }
it { is_expected.to be_allowed(:read_package) }
it { is_expected.to be_allowed(:read_group) }
it { is_expected.to be_allowed(:read_crm_organization) }
it { is_expected.to be_allowed(:read_crm_contact) }
it { is_expected.to be_disallowed(:destroy_package) }
end
@ -967,7 +963,7 @@ RSpec.describe GroupPolicy do
it_behaves_like 'Self-managed Core resource access tokens'
context 'support bot' do
let_it_be(:group) { create(:group, :private) }
let_it_be(:group) { create(:group, :private, :crm_enabled) }
let_it_be(:current_user) { User.support_bot }
before do
@ -977,7 +973,7 @@ RSpec.describe GroupPolicy do
it { expect_disallowed(:read_label) }
context 'when group hierarchy has a project with service desk enabled' do
let_it_be(:subgroup) { create(:group, :private, parent: group) }
let_it_be(:subgroup) { create(:group, :private, :crm_enabled, parent: group) }
let_it_be(:project) { create(:project, group: subgroup, service_desk_enabled: true) }
it { expect_allowed(:read_label) }
@ -1170,7 +1166,7 @@ RSpec.describe GroupPolicy do
end
context 'when crm_enabled is false' do
let(:group) { create(:group) }
let(:group) { create(:group, :crm_enabled) }
let(:current_user) { owner }
it { is_expected.to be_disallowed(:read_crm_contact) }

1
spec/presenters/packages/npm/package_presenter_spec.rb
View file

@ -108,6 +108,7 @@ RSpec.describe ::Packages::Npm::PackagePresenter do
context 'with packages_installable_package_files disabled' do
before do
stub_feature_flags(packages_installable_package_files: false)
package2.package_files.id_not_in(package_file_pending_destruction.id).delete_all
end
it 'returns them' do

2
spec/requests/groups/crm/contacts_controller_spec.rb
View file

@ -73,7 +73,7 @@ RSpec.describe Groups::Crm::ContactsController do
let(:group) { create(:group, :public, :crm_enabled) }
context 'with anonymous user' do
it_behaves_like 'ok response with index template'
it_behaves_like 'response with 404 status'
end
end
end

2
spec/requests/groups/crm/organizations_controller_spec.rb
View file

@ -73,7 +73,7 @@ RSpec.describe Groups::Crm::OrganizationsController do
let(:group) { create(:group, :public, :crm_enabled) }
context 'with anonymous user' do
it_behaves_like 'ok response with index template'
it_behaves_like 'response with 404 status'
end
end
end

103
spec/requests/projects/google_cloud/deployments_controller_spec.rb Normal file
View file

@ -0,0 +1,103 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::GoogleCloud::DeploymentsController do
let_it_be(:project) { create(:project, :public) }
let_it_be(:user_guest) { create(:user) }
let_it_be(:user_developer) { create(:user) }
let_it_be(:user_maintainer) { create(:user) }
let_it_be(:user_creator) { project.creator }
let_it_be(:unauthorized_members) { [user_guest, user_developer] }
let_it_be(:authorized_members) { [user_maintainer, user_creator] }
let_it_be(:urls_list) { %W[#{project_google_cloud_deployments_cloud_run_path(project)} #{project_google_cloud_deployments_cloud_storage_path(project)}] }
before do
project.add_guest(user_guest)
project.add_developer(user_developer)
project.add_maintainer(user_maintainer)
end
describe "Routes must be restricted behind Google OAuth2" do
context 'when a public request is made' do
it 'returns not found on GET request' do
urls_list.each do |url|
get url
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
context 'when unauthorized members make requests' do
it 'returns not found on GET request' do
urls_list.each do |url|
unauthorized_members.each do |unauthorized_member|
sign_in(unauthorized_member)
get url
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
end
context 'when authorized members make requests' do
it 'redirects on GET request' do
urls_list.each do |url|
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to redirect_to(assigns(:authorize_url))
end
end
end
end
end
describe 'Authorized GET project/-/google_cloud/deployments/cloud_run' do
let_it_be(:url) { "#{project_google_cloud_deployments_cloud_run_path(project)}" }
before do
allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client|
allow(client).to receive(:validate_token).and_return(true)
end
end
it 'renders placeholder' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to have_gitlab_http_status(:ok)
end
end
end
describe 'Authorized GET project/-/google_cloud/deployments/cloud_storage' do
let_it_be(:url) { "#{project_google_cloud_deployments_cloud_storage_path(project)}" }
before do
allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client|
allow(client).to receive(:validate_token).and_return(true)
end
end
it 'renders placeholder' do
authorized_members.each do |authorized_member|
sign_in(authorized_member)
get url
expect(response).to have_gitlab_http_status(:ok)
end
end
end
end

18
spec/services/ci/process_sync_events_service_spec.rb
View file

@ -71,6 +71,24 @@ RSpec.describe Ci::ProcessSyncEventsService do
expect { execute }.not_to change(Projects::SyncEvent, :count)
end
end
it 'does not delete non-executed events' do
new_project = create(:project)
sync_event_class.delete_all
project1.update!(group: parent_group_2)
new_project.update!(group: parent_group_1)
project2.update!(group: parent_group_1)
new_project_sync_event = new_project.sync_events.last
allow(sync_event_class).to receive(:preload_synced_relation).and_return(
sync_event_class.where.not(id: new_project_sync_event)
)
expect { execute }.to change(Projects::SyncEvent, :count).from(3).to(1)
expect(new_project_sync_event.reload).to be_persisted
end
end
context 'for Namespaces::SyncEvent' do

8
spec/services/issues/update_service_spec.rb
View file

@ -22,10 +22,10 @@ RSpec.describe Issues::UpdateService, :mailer do
end
before_all do
project.add_maintainer(user)
project.add_developer(user2)
project.add_developer(user3)
project.add_guest(guest)
group.add_maintainer(user)
group.add_developer(user2)
group.add_developer(user3)
group.add_guest(guest)
end
describe 'execute' do

2
spec/support/shared_contexts/policies/group_policy_shared_context.rb
View file

@ -28,6 +28,8 @@ RSpec.shared_context 'GroupPolicy context' do
read_metrics_dashboard_annotation
read_prometheus
read_package_settings
read_crm_contact
read_crm_organization
]
end