kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-08-11 06:12:18 +00:00
parent 8754d20bbb
commit 20d8491f30
24 changed files with 744 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app/assets/javascripts/related_issues/components/related_issues_block.vue
View File

@ -1,5 +1,6 @@
<script>
import { GlLink, GlIcon, GlButton } from '@gitlab/ui';
import { __ } from '~/locale';
import {
issuableIconMap,
issuableQaClassMap,
@ -96,6 +97,11 @@ export default {
default: true,
},
},
data() {
return {
isOpen: true,
};
},
computed: {
hasRelatedIssues() {
return this.relatedIssues.length > 0;
@ -139,6 +145,17 @@ export default {
qaClass() {
return issuableQaClassMap[this.issuableType];
},
toggleIcon() {
return this.isOpen ? 'chevron-lg-up' : 'chevron-lg-down';
},
toggleLabel() {
return this.isOpen ? __('Collapse') : __('Expand');
},
},
methods: {
handleToggle() {
this.isOpen = !this.isOpen;
},
},
linkedIssueTypesTextMap,
};
@ -148,8 +165,8 @@ export default {
<div id="related-issues" class="related-issues-block gl-mt-5">
<div class="card card-slim gl-overflow-hidden">
<div
:class="{ 'panel-empty-heading border-bottom-0': !hasBody }"
class="card-header gl-display-flex gl-justify-content-space-between gl-bg-gray-10"
:class="{ 'panel-empty-heading border-bottom-0': !hasBody, 'gl-border-b-0': !isOpen }"
class="card-header gl-display-flex gl-justify-content-space-between gl-align-items-center gl-bg-gray-10"
>
<h3
class="card-title h5 position-relative gl-my-0 gl-display-flex gl-align-items-center gl-h-7"
@ -182,6 +199,7 @@ export default {
<gl-button
v-if="canAdmin"
data-qa-selector="related_issues_plus_button"
data-testid="add-button"
icon="plus"
:aria-label="addIssuableButtonText"
:class="qaClass"
@ -190,12 +208,23 @@ export default {
</div>
</h3>
<slot name="header-actions"></slot>
<div class="gl-pl-4 gl-ml-3">
<gl-button
category="tertiary"
:icon="toggleIcon"
:aria-label="toggleLabel"
data-testid="toggle-links"
@click="handleToggle"
/>
</div>
</div>
<div
v-if="isOpen"
class="linked-issues-card-body gl-bg-gray-10"
:class="{
'gl-p-5': isFormVisible || shouldShowTokenBody,
}"
data-testid="related-issues-body"
>
<div
v-if="isFormVisible"

10
app/assets/javascripts/work_items/components/work_item_description.vue
View File

@ -172,7 +172,7 @@ export default {
<template>
<gl-form-group
v-if="isEditing"
class="gl-my-5"
class="gl-my-5 gl-border-t gl-pt-6"
:label="__('Description')"
label-for="work-item-description"
>
@ -182,7 +182,7 @@ export default {
:is-submitting="isSubmitting"
:markdown-preview-path="markdownPreviewPath"
:markdown-docs-path="$options.markdownDocsPath"
class="gl-p-3 bordered-box"
class="gl-p-3 bordered-box gl-mt-5"
>
<template #textarea>
<textarea
@ -217,9 +217,9 @@ export default {
}}</gl-button>
</div>
</gl-form-group>
<div v-else class="gl-mb-5">
<div class="gl-display-flex gl-align-items-center gl-mb-5">
<h3 class="gl-font-base gl-my-0">{{ __('Description') }}</h3>
<div v-else class="gl-mb-5 gl-border-t">
<div class="gl-display-inline-flex gl-align-items-center gl-mb-5">
<label class="d-block col-form-label gl-mr-5">{{ __('Description') }}</label>
<gl-button
v-if="canEdit"
class="gl-ml-auto"

65
app/assets/stylesheets/page_bundles/group.scss
View File

@ -72,36 +72,43 @@
}
}
.group-nav-container .nav-controls {
.group-filter-form {
flex: 1 1 auto;
margin-right: $gl-padding-8;
.group-nav-container {
.nav-controls {
.group-filter-form {
flex: 1 1 auto;
margin-right: $gl-padding-8;
}
.dropdown-menu-right {
margin-top: 0;
}
@include media-breakpoint-down(sm) {
.dropdown,
.dropdown .dropdown-toggle,
.btn-success {
display: block;
}
.group-filter-form,
.dropdown {
margin-bottom: 10px;
margin-right: 0;
}
&,
.group-filter-form,
.group-filter-form-field,
.dropdown,
.dropdown .dropdown-toggle,
.btn-success {
width: 100%;
}
}
}
.dropdown-menu-right {
margin-top: 0;
}
@include media-breakpoint-down(sm) {
.dropdown,
.dropdown .dropdown-toggle,
.btn-success {
display: block;
}
.group-filter-form,
.dropdown {
margin-bottom: 10px;
margin-right: 0;
}
&,
.group-filter-form,
.group-filter-form-field,
.dropdown,
.dropdown .dropdown-toggle,
.btn-success {
width: 100%;
}
// Remove this selector once https://gitlab.com/gitlab-org/gitlab/-/issues/370050 is addressed.
.scrolling-tabs-container {
width: 100%;
}
}

18
app/models/namespaces/traversal/linear_scopes.rb
View File

@ -144,25 +144,15 @@ module Namespaces
end
def self_and_descendants_with_comparison_operators(include_self: true)
base = all.select(:traversal_ids)
base = base.select(:id) if Feature.enabled?(:linear_scopes_superset)
base = all.select(:id, :traversal_ids)
base_cte = base.as_cte(:descendants_base_cte)
namespaces = Arel::Table.new(:namespaces)
withs = [base_cte.to_arel]
froms = []
if Feature.enabled?(:linear_scopes_superset)
superset_cte = self.superset_cte(base_cte.table.name)
withs += [superset_cte.to_arel]
froms = [superset_cte.table]
else
froms = [base_cte.table]
end
superset_cte = self.superset_cte(base_cte.table.name)
withs = [base_cte.to_arel, superset_cte.to_arel]
# Order is important. namespace should be last to handle future joins.
froms += [namespaces]
froms = [superset_cte.table, namespaces]
base_ref = froms.first

10
config/feature_flags/development/linear_scopes_superset.yml → config/feature_flags/development/import_export_web_upload_stream.yml
View File

@ -1,8 +1,8 @@
---
name: linear_scopes_superset
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87643
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/362687
milestone: '15.1'
name: import_export_web_upload_stream
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93379
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/370127
milestone: '15.3'
type: development
group: group::workspace
group: group::import
default_enabled: false

7
data/deprecations/15-3-omniauth-cas3.yml
View File

@ -1,7 +1,7 @@
#
# REQUIRED FIELDS
#
- name: "`omniauth-cas3` gem" # (required) The name of the feature to be deprecated
- name: "CAS OmniAuth provider" # (required) The name of the feature to be deprecated
announcement_milestone: "15.3" # (required) The milestone when this feature was first announced as deprecated.
announcement_date: "2022-08-22" # (required) The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
removal_milestone: "16.0" # (required) The milestone when this feature is planned to be removed
@ -11,5 +11,6 @@
stage: Manage # (required) String value of the stage that the feature was created in. e.g., Growth
issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/369127 # (required) Link to the deprecation issue in GitLab
body: | # (required) Do not modify this line, instead modify the lines below.
The `omniauth-cas3` gem will be removed in our next major release, GitLab 16.0. This gem sees very little use and
its lack of upstream maintence is preventing GitLab's [upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).
The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider will be removed in our next major
release, GitLab 16.0. This gem sees very little use and its lack of upstream maintenance is preventing GitLab's
[upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).

9
data/deprecations/15-3-omniauth-crowd.yml
View File

@ -1,7 +1,7 @@
#
# REQUIRED FIELDS
#
- name: "`omniauth_crowd` gem" # (required) The name of the feature to be deprecated
- name: "Atlassian Crowd OmniAuth provider" # (required) The name of the feature to be deprecated
announcement_milestone: "15.3" # (required) The milestone when this feature was first announced as deprecated.
announcement_date: "2022-08-22" # (required) The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
removal_milestone: "16.0" # (required) The milestone when this feature is planned to be removed
@ -11,6 +11,7 @@
stage: Manage # (required) String value of the stage that the feature was created in. e.g., Growth
issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/369117 # (required) Link to the deprecation issue in GitLab
body: | # (required) Do not modify this line, instead modify the lines below.
The `omniauth_crowd` gem will be removed in our next major release, GitLab 16.0. This gem sees very little use and
its [lack of compatability](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is blocking
our upgrade.
The `omniauth_crowd` gem that provides GitLab with the Atlassian Crowd OmniAuth provider will be removed in our
next major release, GitLab 16.0. This gem sees very little use and its
[lack of compatibility](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is
[blocking our upgrade](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).

2
data/deprecations/15-3-pipeline_activity_limit.yml
View File

@ -27,7 +27,7 @@
stage: Verify # (required) String value of the stage that the feature was created in. e.g., Growth
issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/368195 # (required) Link to the deprecation issue in GitLab
body: | # (required) Do not modify this line, instead modify the lines below.
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the Rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html).
- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits).

6
doc/administration/auth/crowd.md
View File

@ -5,7 +5,11 @@ group: Authentication and Authorization
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Atlassian Crowd OmniAuth Provider **(FREE SELF)**
# Atlassian Crowd OmniAuth provider (deprecated) **(FREE SELF)**
WARNING:
This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369117) in GitLab 15.3 and is planned for
removal in 16.0.
Authenticate to GitLab using the Atlassian Crowd OmniAuth provider. Enabling
this provider also allows Crowd authentication for Git-over-https requests.

16
doc/administration/gitaly/configure_gitaly.md
View File

@ -883,13 +883,15 @@ gitaly['cgroups_repositories_cpu_shares'] => 512
on the repository the command is for. A circular hashing algorithm assigns
Git commands to these cgroups, so a Git command for a repository is
always assigned to the same cgroup.
- `cgroups_repositories_memory_bytes` is the total memory limit that is imposed collectively on all
Git processes that Gitaly spawns. 0 implies no limit. This value cannot exceed
that of the top level `cgroups_memory_bytes`.
- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed collectively on all Git
processes Gitaly spawns. 0 implies no limit. The maximum is 1024 shares,
which represents 100% of CPU. This value cannot exceed that of the top
level`cgroups_cpu_shares`.
- `cgroups_repositories_memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup.
A repository cgroup is one that contains Git processes for one or more repositories.
A consistent hash is used to assign repositories to these cgroups such that a Git process for a given repository always ends up in the same cgroup.
0 implies no limit. This value cannot exceed that of the top level `cgroups_memory_bytes`.
- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup.
A repository cgroup is one that contains Git processes for one or more repositories.
A consistent hash is used to assign repositories to these cgroups such that a Git process for a given repository always ends up in the same cgroup.
0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU.
This value cannot exceed that of the top level`cgroups_cpu_shares`.
#### Configure cgroups (legacy method)

6
doc/integration/cas.md
View File

@ -4,7 +4,11 @@ group: Authentication and Authorization
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# CAS OmniAuth Provider **(FREE SELF)**
# CAS OmniAuth provider (deprecated) **(FREE SELF)**
WARNING:
This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369127) in GitLab 15.3 and is planned for
removal in 16.0.
To enable the CAS OmniAuth provider you must register your application with your
CAS instance. This requires the service URL GitLab supplies to CAS. It should be

1
doc/integration/index.md
View File

@ -20,7 +20,6 @@ GitLab can be configured to authenticate access requests with the following auth
- Enable the [Auth0 OmniAuth](auth0.md) provider.
- Enable sign in with [Bitbucket](bitbucket.md) accounts.
- Configure GitLab to sign in using [CAS](cas.md).
- Integrate with [Kerberos](kerberos.md).
- Enable sign in via [LDAP](../administration/auth/ldap/index.md).
- Enable [OAuth2 provider](oauth_provider.md) application creation.

13
doc/integration/omniauth.md
View File

@ -18,7 +18,6 @@ GitLab supports the following OmniAuth providers.
| Provider documentation | OmniAuth provider name |
|---------------------------------------------------------------------|----------------------------|
| [AliCloud](alicloud.md) | `alicloud` |
| [Atlassian Crowd](../administration/auth/crowd.md) | `crowd` |
| [Atlassian](../administration/auth/atlassian.md) | `atlassian_oauth2` |
| [Auth0](auth0.md) | `auth0` |
| [Authentiq](../administration/auth/authentiq.md) | `authentiq` |
@ -26,7 +25,6 @@ GitLab supports the following OmniAuth providers.
| [Azure v2](azure.md) | `azure_activedirectory_v2` |
| [Azure v1](azure.md) | `azure_oauth2` |
| [Bitbucket Cloud](bitbucket.md) | `bitbucket` |
| [CAS](cas.md) | `cas3` |
| [DingTalk](ding_talk.md) | `dingtalk` |
| [Facebook](facebook.md) | `facebook` |
| [Generic OAuth 2.0](oauth2_generic.md) | `oauth2_generic` |
@ -49,7 +47,7 @@ Setting | Description | Default value
---------------------------|-------------|--------------
`allow_single_sign_on` | Enables you to list the providers that automatically create a GitLab account. The provider names are available in the **OmniAuth provider name** column in the [supported providers table](#supported-providers). | The default is `false`. If `false`, users must be created manually, or they can't sign in using OmniAuth.
`auto_link_ldap_user` | If enabled, creates an LDAP identity in GitLab for users that are created through an OmniAuth provider. You can enable this setting if you have the [LDAP (ActiveDirectory)](../administration/auth/ldap/index.md) integration enabled. Requires the `uid` of the user to be the same in both LDAP and the OmniAuth provider. | The default is `false`.
`block_auto_created_users` | If enabled, blocks users that are automatically created from signing in until they are approved by an administrator. | The default is `true`. If you set the value to `false`, make sure you only define providers for `allow_single_sign_on` that you can control, like SAML, Crowd, or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval.
`block_auto_created_users` | If enabled, blocks users that are automatically created from signing in until they are approved by an administrator. | The default is `true`. If you set the value to `false`, make sure you only define providers for `allow_single_sign_on` that you can control, like SAML or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval.
To change these settings:
@ -134,7 +132,7 @@ variable in `args` for a provider, you can select another claim to use for the G
args: { gitlab_username_claim: 'sub' } # For users signing in with the provider you configure, the GitLab username will be set to the "sub" received from the provider
},
# Here are examples using GitHub and Crowd
# Here are examples using GitHub and Kerberos
gitlab_rails['omniauth_providers'] = {
name: "github"
@ -142,9 +140,9 @@ variable in `args` for a provider, you can select another claim to use for the G
args: { gitlab_username_claim: 'name' } # For users signing in with GitHub, the GitLab username will be set to the "name" received from GitHub
},
{
name: "crowd"
name: "kerberos"
...
args: { gitlab_username_claim: 'uid' } # For users signing in with Crowd, the GitLab username will be set to the "uid" received from Crowd
args: { gitlab_username_claim: 'uid' } # For users signing in with Kerberos, the GitLab username will be set to the "uid" received from Kerberos
},
]
```
@ -160,7 +158,7 @@ variable in `args` for a provider, you can select another claim to use for the G
...
args: { gitlab_username_claim: 'name' }
}
- { name: 'crowd',
- { name: 'kerberos',
...
args: { gitlab_username_claim: 'uid' }
}
@ -442,5 +440,4 @@ then override the icon in one of two ways:
## Limitations
Most supported OmniAuth providers don't support Git over HTTP password authentication.
The only exception is [Atlassian Crowd](../administration/auth/crowd.md) (since GitLab [13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46935)).
As a workaround, you can authenticate using a [personal access token](../user/profile/personal_access_tokens.md).

66
doc/update/deprecations.md
View File

@ -47,48 +47,50 @@ sole discretion of GitLab Inc.
## Announced in 15.3
<div class="deprecation removal-160 breaking-change">
### Atlassian Crowd OmniAuth provider
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
WARNING:
This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
Review the details carefully before upgrading.
The `omniauth_crowd` gem that provides GitLab with the Atlassian Crowd OmniAuth provider will be removed in our
next major release, GitLab 16.0. This gem sees very little use and its
[lack of compatibility](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is
[blocking our upgrade](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).
</div>
<div class="deprecation removal-160 breaking-change">
### CAS OmniAuth provider
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
WARNING:
This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
Review the details carefully before upgrading.
The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider will be removed in our next major
release, GitLab 16.0. This gem sees very little use and its lack of upstream maintenance is preventing GitLab's
[upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).
</div>
<div class="deprecation removal-160">
### Maximum number of active pipelines per project limit (`ci_active_pipelines`)
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the Rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection:
- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html).
- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits).
</div>
<div class="deprecation removal-160 breaking-change">
### `omniauth-cas3` gem
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
WARNING:
This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
Review the details carefully before upgrading.
The `omniauth-cas3` gem will be removed in our next major release, GitLab 16.0. This gem sees very little use and
its lack of upstream maintence is preventing GitLab's [upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).
</div>
<div class="deprecation removal-160 breaking-change">
### `omniauth_crowd` gem
Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)
WARNING:
This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
Review the details carefully before upgrading.
The `omniauth_crowd` gem will be removed in our next major release, GitLab 16.0. This gem sees very little use and
its [lack of compatability](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is blocking
our upgrade.
</div>
</div>

2
lib/gitlab/database/migrations/batched_background_migration_helpers.rb
View File

@ -24,7 +24,7 @@ module Gitlab
# class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be
# present in the Gitlab::BackgroundMigration::BatchingStrategies module.
#
# If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper
# If migration with same job_class_name, table_name, column_name, and job_arguments already exists, this helper
# will log an warning and not create a new one.
#
# job_class_name - The background migration job class as a string

19
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
View File

@ -12,12 +12,13 @@ module Gitlab
private
attr_reader :project, :current_user, :lock_file
attr_reader :project, :current_user, :lock_file, :logger
public
def initialize(attributes = {})
@options = attributes
@logger = Gitlab::Export::Logger.build
end
def method_missing(method, *args)
@ -43,6 +44,10 @@ module Gitlab
true
rescue StandardError => e
payload = { message: "After export strategy failed" }
Gitlab::ExceptionLogFormatter.format!(e, payload)
log_error(payload)
project.import_export_shared.error(e)
false
ensure
@ -108,6 +113,18 @@ module Gitlab
def log_validation_errors
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end
def log_info(params)
logger.info(log_default_params.merge(params))
end
def log_error(params)
logger.error(log_default_params.merge(params))
end
def log_default_params
{ project_name: project.name, project_id: project.id }
end
end
end
end

1
lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
View File

@ -5,6 +5,7 @@ module Gitlab
module AfterExportStrategies
class MoveFileStrategy < BaseAfterExportStrategy
def initialize(archive_path:)
super
@archive_path = archive_path
end

28
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
View File

@ -23,7 +23,17 @@ module Gitlab
protected
def strategy_execute
handle_response_error(send_file)
log_info(message: "Started uploading project", export_size: export_size)
upload_duration = Benchmark.realtime do
if Feature.enabled?(:import_export_web_upload_stream) && !project.export_file.file_storage?
upload_project_as_remote_stream
else
handle_response_error(send_file)
end
end
log_info(message: "Finished uploading project", export_size: export_size, upload_duration: upload_duration)
end
def handle_response_error(response)
@ -44,8 +54,22 @@ module Gitlab
export_file.close if export_file
end
def upload_project_as_remote_stream
Gitlab::ImportExport::RemoteStreamUpload.new(
download_url: project.export_file.url,
upload_url: url,
options: {
upload_method: http_method.downcase.to_sym,
upload_content_type: 'application/gzip'
}).execute
rescue Gitlab::ImportExport::RemoteStreamUpload::StreamError => e
log_error(message: e.message, response_body: e.response_body.truncate(3000))
raise
end
def export_file
project.export_file.open
@export_file ||= project.export_file.open
end
def send_file_options

117
lib/gitlab/import_export/remote_stream_upload.rb Normal file
View File

@ -0,0 +1,117 @@
# frozen_string_literal: true
# This class downloads a file from one URL and uploads it to another URL
# without having to save the file on the disk and loading the whole file in
# memory. The download and upload are performed in chunks size of
# `buffer_size`. A chunk is downloaded, then uploaded, then a next chunk is
# downloaded and uploaded. This repeats until all the file is processed.
module Gitlab
module ImportExport
class RemoteStreamUpload
def initialize(download_url:, upload_url:, options: {})
@download_url = download_url
@upload_url = upload_url
@upload_method = options[:upload_method] || :post
@upload_content_type = options[:upload_content_type] || 'application/gzip'
end
def execute
receive_data(download_url) do |response, chunks|
send_data(upload_url, response.content_length, chunks) do |response|
if response.code != '200'
raise StreamError.new("Invalid response code while uploading file. Code: #{response.code}", response.body)
end
end
end
end
class StreamError < StandardError
attr_reader :response_body
def initialize(message, response_body = '')
super(message)
@response_body = response_body
end
end
class ChunkStream
DEFAULT_BUFFER_SIZE = 128.kilobytes
def initialize(chunks)
@chunks = chunks
@last_chunk = nil
@end_of_chunks = false
end
def read(n1 = nil, n2 = nil)
ensure_chunk&.read(n1, n2)
end
private
def ensure_chunk
return @last_chunk if @last_chunk && !@last_chunk.eof?
return if @end_of_chunks
@last_chunk = read_next_chunk
end
def read_next_chunk
next_chunk = StringIO.new
begin
next_chunk.write(@chunks.next) until next_chunk.size > DEFAULT_BUFFER_SIZE
rescue StopIteration
@end_of_chunks = true
end
next_chunk.rewind
next_chunk
end
end
private
attr_reader :download_url, :upload_url, :upload_method, :upload_content_type, :logger
def receive_data(uri)
http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection
http.start do
request = Net::HTTP::Get.new(uri)
http.request(request) do |response|
if response.code == '200'
yield(response, response.enum_for(:read_body))
else
raise StreamError.new(
"Invalid response code while downloading file. Code: #{response.code}",
response.body
)
end
end
end
end
def send_data(uri, content_length, chunks)
http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection
http.start do
request = upload_request_class(upload_method).new(uri)
request.body_stream = ChunkStream.new(chunks)
request.content_length = content_length
request.content_type = upload_content_type
http.request(request) do |response|
yield(response)
end
end
end
def upload_request_class(upload_method)
return Net::HTTP::Put if upload_method == :put
Net::HTTP::Post
end
end
end
end

2
locale/gitlab.pot
View File

@ -11627,7 +11627,7 @@ msgstr ""
msgid "DORA4Metrics|Change failure rate"
msgstr ""
msgid "DORA4Metrics|Change failure rate (%%)"
msgid "DORA4Metrics|Change failure rate (percentage)"
msgstr ""
msgid "DORA4Metrics|Date"

51
spec/frontend/issuable/related_issues/components/related_issues_block_spec.js
View File

@ -1,5 +1,6 @@
import { GlButton, GlIcon } from '@gitlab/ui';
import { shallowMount, mount } from '@vue/test-utils';
import { nextTick } from 'vue';
import { GlIcon } from '@gitlab/ui';
import { mountExtended, shallowMountExtended } from 'helpers/vue_test_utils_helper';
import {
issuable1,
issuable2,
@ -17,7 +18,9 @@ import {
describe('RelatedIssuesBlock', () => {
let wrapper;
const findIssueCountBadgeAddButton = () => wrapper.find(GlButton);
const findIssueCountBadgeAddButton = () => wrapper.findByTestId('add-button');
const findToggleButton = () => wrapper.findByTestId('toggle-links');
const findRelatedIssuesBody = () => wrapper.findByTestId('related-issues-body');
afterEach(() => {
if (wrapper) {
@ -28,7 +31,7 @@ describe('RelatedIssuesBlock', () => {
describe('with defaults', () => {
beforeEach(() => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
issuableType: issuableTypesMap.ISSUE,
@ -43,7 +46,7 @@ describe('RelatedIssuesBlock', () => {
`(
'displays "$titleText" in the header, "$helpLinkText" aria-label for help link, and "$addButtonText" aria-label for add button when issuableType is set to "$issuableType"',
({ issuableType, pathIdSeparator, titleText, helpLinkText, addButtonText }) => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator,
issuableType,
@ -73,7 +76,7 @@ describe('RelatedIssuesBlock', () => {
it('displays header text slot data', () => {
const headerText = '<div>custom header text</div>';
wrapper = shallowMount(RelatedIssuesBlock, {
wrapper = shallowMountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
issuableType: 'issue',
@ -89,7 +92,7 @@ describe('RelatedIssuesBlock', () => {
it('displays header actions slot data', () => {
const headerActions = '<button data-testid="custom-button">custom button</button>';
wrapper = shallowMount(RelatedIssuesBlock, {
wrapper = shallowMountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
issuableType: 'issue',
@ -103,7 +106,7 @@ describe('RelatedIssuesBlock', () => {
describe('with isFetching=true', () => {
beforeEach(() => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
isFetching: true,
@ -119,7 +122,7 @@ describe('RelatedIssuesBlock', () => {
describe('with canAddRelatedIssues=true', () => {
beforeEach(() => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
canAdmin: true,
@ -135,7 +138,7 @@ describe('RelatedIssuesBlock', () => {
describe('with isFormVisible=true', () => {
beforeEach(() => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
isFormVisible: true,
@ -159,7 +162,7 @@ describe('RelatedIssuesBlock', () => {
const categorizedHeadings = () => wrapper.findAll('h4');
const headingTextAt = (index) => categorizedHeadings().at(index).text();
const mountComponent = (showCategorizedIssues) => {
wrapper = mount(RelatedIssuesBlock, {
wrapper = mountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
relatedIssues: [issuable1, issuable2, issuable3],
@ -217,7 +220,7 @@ describe('RelatedIssuesBlock', () => {
},
].forEach(({ issuableType, icon }) => {
it(`issuableType=${issuableType} is passed`, () => {
wrapper = shallowMount(RelatedIssuesBlock, {
wrapper = shallowMountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
issuableType,
@ -230,4 +233,28 @@ describe('RelatedIssuesBlock', () => {
});
});
});
describe('toggle', () => {
beforeEach(() => {
wrapper = shallowMountExtended(RelatedIssuesBlock, {
propsData: {
pathIdSeparator: PathIdSeparator.Issue,
issuableType: issuableTypesMap.ISSUE,
},
});
});
it('is expanded by default', () => {
expect(findToggleButton().props('icon')).toBe('chevron-lg-up');
expect(findRelatedIssuesBody().exists()).toBe(true);
});
it('expands on click toggle button', async () => {
findToggleButton().vm.$emit('click');
await nextTick();
expect(findToggleButton().props('icon')).toBe('chevron-lg-down');
expect(findRelatedIssuesBody().exists()).toBe(false);
});
});
});

170
spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
View File

@ -9,12 +9,21 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
allow_next_instance_of(ProjectExportWorker) do |job|
allow(job).to receive(:jid).and_return(SecureRandom.hex(8))
end
stub_feature_flags(import_export_web_upload_stream: false)
stub_uploads_object_storage(FileUploader, enabled: false)
end
let(:example_url) { 'http://www.example.com' }
let(:strategy) { subject.new(url: example_url, http_method: 'post') }
let!(:project) { create(:project, :with_export) }
let!(:user) { build(:user) }
let(:user) { build(:user) }
let(:project) { import_export_upload.project }
let(:import_export_upload) do
create(
:import_export_upload,
export_file: fixture_file_upload('spec/fixtures/gitlab/import_export/lightweight_project_export.tar.gz')
)
end
subject { described_class }
@ -36,21 +45,43 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
describe '#execute' do
context 'when upload succeeds' do
before do
allow(strategy).to receive(:send_file)
allow(strategy).to receive(:handle_response_error)
stub_full_request(example_url, method: :post).to_return(status: 200)
end
it 'does not remove the exported project file after the upload' do
it 'does not remove the exported project file after the upload', :aggregate_failures do
expect(project).not_to receive(:remove_exports)
strategy.execute(user, project)
end
it 'has finished export status' do
strategy.execute(user, project)
expect { strategy.execute(user, project) }.not_to change(project, :export_status)
expect(project.export_status).to eq(:finished)
end
it 'logs when upload starts and finishes' do
export_size = import_export_upload.export_file.size
expect_next_instance_of(Gitlab::Export::Logger) do |logger|
expect(logger).to receive(:info).ordered.with(
{
message: "Started uploading project",
project_id: project.id,
project_name: project.name,
export_size: export_size
}
)
expect(logger).to receive(:info).ordered.with(
{
message: "Finished uploading project",
project_id: project.id,
project_name: project.name,
export_size: export_size,
upload_duration: anything
}
)
end
strategy.execute(user, project)
end
end
context 'when upload fails' do
@ -64,5 +95,124 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
expect(errors.first).to eq "Error uploading the project. Code 404: Page not found"
end
end
context 'when object store is disabled' do
it 'reads file from disk and uploads to external url' do
stub_request(:post, example_url).to_return(status: 200)
expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new)
expect(Gitlab::HttpIO).not_to receive(:new)
strategy.execute(user, project)
expect(a_request(:post, example_url)).to have_been_made
end
end
context 'when object store is enabled' do
before do
object_store_url = 'http://object-storage/project.tar.gz'
stub_uploads_object_storage(FileUploader)
stub_request(:get, object_store_url)
stub_request(:post, example_url)
allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url)
allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false)
end
it 'reads file using Gitlab::HttpIO and uploads to external url' do
expect_next_instance_of(Gitlab::HttpIO) do |http_io|
expect(http_io).to receive(:read).and_call_original
end
expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new)
strategy.execute(user, project)
expect(a_request(:post, example_url)).to have_been_made
end
end
context 'when `import_export_web_upload_stream` feature is enabled' do
before do
stub_feature_flags(import_export_web_upload_stream: true)
end
context 'when remote object store is disabled' do
it 'reads file from disk and uploads to external url' do
stub_request(:post, example_url).to_return(status: 200)
expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new)
expect(Gitlab::HttpIO).not_to receive(:new)
strategy.execute(user, project)
expect(a_request(:post, example_url)).to have_been_made
end
end
context 'when object store is enabled' do
let(:object_store_url) { 'http://object-storage/project.tar.gz' }
before do
stub_uploads_object_storage(FileUploader)
allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url)
allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false)
end
it 'uploads file as a remote stream' do
arguments = {
download_url: object_store_url,
upload_url: example_url,
options: {
upload_method: :post,
upload_content_type: 'application/gzip'
}
}
expect_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload, arguments) do |remote_stream_upload|
expect(remote_stream_upload).to receive(:execute)
end
expect(Gitlab::HttpIO).not_to receive(:new)
strategy.execute(user, project)
end
context 'when upload as remote stream raises an exception' do
before do
allow_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload) do |remote_stream_upload|
allow(remote_stream_upload).to receive(:execute).and_raise(
Gitlab::ImportExport::RemoteStreamUpload::StreamError.new('Exception error message', 'Response body')
)
end
end
it 'logs the exception and stores the error message' do
expect_next_instance_of(Gitlab::Export::Logger) do |logger|
expect(logger).to receive(:error).ordered.with(
{
project_id: project.id,
project_name: project.name,
message: 'Exception error message',
response_body: 'Response body'
}
)
expect(logger).to receive(:error).ordered.with(
{
project_id: project.id,
project_name: project.name,
message: 'After export strategy failed',
'exception.class' => 'Gitlab::ImportExport::RemoteStreamUpload::StreamError',
'exception.message' => 'Exception error message',
'exception.backtrace' => anything
}
)
end
strategy.execute(user, project)
expect(project.import_export_shared.errors.first).to eq('Exception error message')
end
end
end
end
end
end

232
spec/lib/gitlab/import_export/remote_stream_upload_spec.rb Normal file
View File

@ -0,0 +1,232 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::ImportExport::RemoteStreamUpload do
include StubRequests
subject do
described_class.new(
download_url: download_url,
upload_url: upload_url,
options: {
upload_method: upload_method,
upload_content_type: upload_content_type
}
)
end
let(:download_url) { 'http://object-storage/file.txt' }
let(:upload_url) { 'http://example.com/file.txt' }
let(:upload_method) { :post }
let(:upload_content_type) { 'text/plain' }
describe '#execute' do
context 'when download request and upload request return 200' do
it 'uploads the downloaded content' do
stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 })
stub_request(:post, upload_url)
subject.execute
expect(
a_request(:post, upload_url).with(
body: 'ABC', headers: { 'Content-Length' => 3, 'Content-Type' => 'text/plain' }
)
).to have_been_made
end
end
context 'when upload method is put' do
let(:upload_method) { :put }
it 'uploads using the put method' do
stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 })
stub_request(:put, upload_url)
subject.execute
expect(
a_request(:put, upload_url).with(
body: 'ABC', headers: { 'Content-Length' => 3, 'Content-Type' => 'text/plain' }
)
).to have_been_made
end
end
context 'when download request does not return 200' do
it do
stub_request(:get, download_url).to_return(status: 404)
expect { subject.execute }.to raise_error(
Gitlab::ImportExport::RemoteStreamUpload::StreamError,
"Invalid response code while downloading file. Code: 404"
)
end
end
context 'when upload request does not returns 200' do
it do
stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 })
stub_request(:post, upload_url).to_return(status: 403)
expect { subject.execute }.to raise_error(
Gitlab::ImportExport::RemoteStreamUpload::StreamError,
"Invalid response code while uploading file. Code: 403"
)
end
end
context 'when download URL is a local address' do
let(:download_url) { 'http://127.0.0.1/file.txt' }
before do
stub_request(:get, download_url)
stub_request(:post, upload_url)
end
it 'raises error' do
expect { subject.execute }.to raise_error(
Gitlab::HTTP::BlockedUrlError,
"URL 'http://127.0.0.1/file.txt' is blocked: Requests to localhost are not allowed"
)
end
context 'when local requests are allowed' do
before do
stub_application_setting(allow_local_requests_from_web_hooks_and_services: true)
end
it 'raises does not error' do
expect { subject.execute }.not_to raise_error
end
end
end
context 'when download URL is a local network' do
let(:download_url) { 'http://172.16.0.0/file.txt' }
before do
stub_request(:get, download_url)
stub_request(:post, upload_url)
end
it 'raises error' do
expect { subject.execute }.to raise_error(
Gitlab::HTTP::BlockedUrlError,
"URL 'http://172.16.0.0/file.txt' is blocked: Requests to the local network are not allowed"
)
end
context 'when local network requests are allowed' do
before do
stub_application_setting(allow_local_requests_from_web_hooks_and_services: true)
end
it 'raises does not error' do
expect { subject.execute }.not_to raise_error
end
end
end
context 'when upload URL is a local address' do
let(:upload_url) { 'http://127.0.0.1/file.txt' }
before do
stub_request(:get, download_url)
stub_request(:post, upload_url)
end
it 'raises error' do
stub_request(:get, download_url)
expect { subject.execute }.to raise_error(
Gitlab::HTTP::BlockedUrlError,
"URL 'http://127.0.0.1/file.txt' is blocked: Requests to localhost are not allowed"
)
end
context 'when local requests are allowed' do
before do
stub_application_setting(allow_local_requests_from_web_hooks_and_services: true)
end
it 'raises does not error' do
expect { subject.execute }.not_to raise_error
end
end
end
context 'when upload URL it is a request to local network' do
let(:upload_url) { 'http://172.16.0.0/file.txt' }
before do
stub_request(:get, download_url)
stub_request(:post, upload_url)
end
it 'raises error' do
expect { subject.execute }.to raise_error(
Gitlab::HTTP::BlockedUrlError,
"URL 'http://172.16.0.0/file.txt' is blocked: Requests to the local network are not allowed"
)
end
context 'when local network requests are allowed' do
before do
stub_application_setting(allow_local_requests_from_web_hooks_and_services: true)
end
it 'raises does not error' do
expect { subject.execute }.not_to raise_error
end
end
end
context 'when upload URL resolves to a local address' do
let(:upload_url) { 'http://example.com/file.txt' }
it 'raises error' do
stub_request(:get, download_url)
stub_full_request(upload_url, ip_address: '127.0.0.1', method: upload_method)
expect { subject.execute }.to raise_error(
Gitlab::HTTP::BlockedUrlError,
"URL 'http://example.com/file.txt' is blocked: Requests to localhost are not allowed"
)
end
end
end
describe Gitlab::ImportExport::RemoteStreamUpload::ChunkStream do
describe 'StringIO#copy_stream compatibility' do
it 'copies all chunks' do
chunks = %w[ABC EFD].to_enum
chunk_stream = described_class.new(chunks)
new_stream = StringIO.new
IO.copy_stream(chunk_stream, new_stream)
new_stream.rewind
expect(new_stream.read).to eq('ABCEFD')
end
context 'with chunks smaller and bigger than buffer size' do
before do
stub_const('Gitlab::ImportExport::RemoteStreamUpload::ChunkStream::DEFAULT_BUFFER_SIZE', 4)
end
it 'copies all chunks' do
chunks = %w[A BC DEF GHIJ KLMNOPQ RSTUVWXYZ].to_enum
chunk_stream = described_class.new(chunks)
new_stream = StringIO.new
IO.copy_stream(chunk_stream, new_stream)
new_stream.rewind
expect(new_stream.read).to eq('ABCDEFGHIJKLMNOPQRSTUVWXYZ')
end
end
end
end
end

16
spec/support/shared_examples/namespaces/traversal_scope_examples.rb
View File

@ -273,14 +273,6 @@ RSpec.shared_examples 'namespace traversal scopes' do
include_examples '.self_and_descendants'
end
context 'with linear_scopes_superset feature flag disabled' do
before do
stub_feature_flags(linear_scopes_superset: false)
end
include_examples '.self_and_descendants'
end
end
shared_examples '.self_and_descendant_ids' do
@ -324,14 +316,6 @@ RSpec.shared_examples 'namespace traversal scopes' do
include_examples '.self_and_descendant_ids'
end
context 'with linear_scopes_superset feature flag disabled' do
before do
stub_feature_flags(linear_scopes_superset: false)
end
include_examples '.self_and_descendant_ids'
end
end
shared_examples '.self_and_hierarchy' do