kotovalexarian-likes-gitlab
/
gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-06-14 18:09:25 +00:00
parent 9b8269e570
commit 505485f485
82 changed files with 1318 additions and 1006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab/CODEOWNERS
View File

@ -604,6 +604,7 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab
/doc/gitlab-basics/ @aqualls
/doc/install/ @axil
/doc/integration/ @kpaizee
/doc/integration/advanced_search/ @sselhorn
/doc/integration/elasticsearch.md @sselhorn
/doc/integration/gitpod.md @aqualls
/doc/integration/kerberos.md @eread

6
.gitlab/ci/rules.gitlab-ci.yml
View File

@ -1076,10 +1076,8 @@
rules:
- <<: *if-merge-request-labels-run-all-rspec
- <<: *if-merge-request
changes: *core-backend-patterns
- <<: *if-merge-request
changes: *ci-patterns
- changes: ["config/**/*"]
changes: *backend-patterns
- changes: *core-backend-patterns
.rails:rules:code-backstage-qa:
rules:

2
Gemfile
View File

@ -170,7 +170,7 @@ gem 'asciidoctor-kroki', '~> 0.5.0', require: false
gem 'rouge', '~> 3.27.0'
gem 'truncato', '~> 0.7.11'
gem 'bootstrap_form', '~> 4.2.0'
gem 'nokogiri', '~> 1.12'
gem 'nokogiri', '~> 1.13.6'
gem 'escape_utils', '~> 1.1'
# Calendar rendering

4
Gemfile.lock
View File

@ -807,7 +807,7 @@ GEM
netrc (0.11.0)
nio4r (2.5.8)
no_proxy_fix (0.1.2)
nokogiri (1.13.3)
nokogiri (1.13.6)
mini_portile2 (~> 2.8.0)
racc (~> 1.4)
notiffany (0.1.3)
@ -1603,7 +1603,7 @@ DEPENDENCIES
multi_json (~> 1.14.1)
net-ldap (~> 0.16.3)
net-ntp
nokogiri (~> 1.12)
nokogiri (~> 1.13.6)
oauth2 (~> 1.4)
octokit (~> 4.15)
ohai (~> 16.10)

13
app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue
View File

@ -8,7 +8,7 @@ import { confidentialityQueries } from '~/sidebar/constants';
export default {
i18n: {
confidentialityOnWarning: __(
'You are going to turn on confidentiality. Only %{context} members with %{strongStart}at least Reporter role%{strongEnd} can view or be notified about this %{issuableType}.',
'You are going to turn on confidentiality. Only %{context} members with %{strongStart}%{permissions}%{strongEnd} can view or be notified about this %{issuableType}.',
),
confidentialityOffWarning: __(
'You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}.',
@ -65,6 +65,11 @@ export default {
groupPath: this.fullPath,
};
},
permissions() {
return this.issuableType === IssuableType.Issue
? __('at least the Reporter role, the author, and assignees')
: __('at least the Reporter role');
},
},
methods: {
submitForm() {
@ -120,7 +125,11 @@ export default {
<p data-testid="warning-message">
<gl-sprintf :message="warningMessage">
<template #strong="{ content }">
<strong>{{ content }}</strong>
<strong>
<gl-sprintf :message="content">
<template #permissions>{{ permissions }}</template>
</gl-sprintf>
</strong>
</template>
<template #context>{{ context }}</template>
<template #issuableType>{{ issuableType }}</template>

6
app/assets/javascripts/vue_shared/constants.js
View File

@ -71,10 +71,14 @@ export const AVATAR_SHAPE_OPTION_RECT = 'rect';
export const confidentialityInfoText = (workspaceType, issuableType) =>
sprintf(
__(
'Only %{workspaceType} members with at least Reporter role can view or be notified about this %{issuableType}.',
'Only %{workspaceType} members with %{permissions} can view or be notified about this %{issuableType}.',
),
{
workspaceType: workspaceType === WorkspaceType.project ? __('project') : __('group'),
issuableType: issuableType === IssuableType.Issue ? __('issue') : __('epic'),
permissions:
issuableType === IssuableType.Issue
? __('at least the Reporter role, the author, and assignees')
: __('at least the Reporter role'),
},
);

8
app/assets/stylesheets/highlight/common.scss
View File

@ -126,3 +126,11 @@
@include gl-display-inline-block;
}
}
@mixin hljs-override($suffix, $color) {
&.blob-viewer {
.hljs-#{$suffix} {
color: $color;
}
}
}

19
app/assets/stylesheets/highlight/themes/monokai.scss
View File

@ -96,6 +96,25 @@ $monokai-gh: #75715e;
}
.code.monokai {
// Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167)
// We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251)
@include hljs-override('string', $monokai-s);
@include hljs-override('attr', $monokai-na);
@include hljs-override('keyword', $monokai-k);
@include hljs-override('variable', $monokai-nv);
@include hljs-override('variable.language_', $monokai-k);
@include hljs-override('title', $monokai-nf);
@include hljs-override('name', $monokai-k);
@include hljs-override('tag', $monokai-nt);
@include hljs-override('type', $monokai-nc);
@include hljs-override('number', $monokai-mf);
@include hljs-override('literal', $monokai-kc);
@include hljs-override('built_in', $monokai-n);
@include hljs-override('section', $monokai-gh);
@include hljs-override('bullet', $monokai-n);
@include hljs-override('subst', $monokai-p);
@include hljs-override('symbol', $monokai-ni);
// Line numbers
.file-line-num {
@include line-number-link($monokai-line-num-color);

8
app/assets/stylesheets/highlight/themes/none.scss
View File

@ -15,6 +15,14 @@
}
.code.none {
// Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167)
// We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251)
&.blob-viewer {
[class^="hljs-"] {
color: $gl-text-color;
}
}
// Line numbers
.file-line-num {
@include line-number-link($black-transparent);

19
app/assets/stylesheets/highlight/themes/solarized-dark.scss
View File

@ -99,6 +99,25 @@ $solarized-dark-il: #2aa198;
}
.code.solarized-dark {
// Highlight.js theme overrides (https://gitlab.com/gitlab-org/gitlab/-/issues/365167)
// We should be able to remove the overrides once the upstream issue is fixed (https://github.com/sourcegraph/sourcegraph/issues/23251)
@include hljs-override('string', $solarized-dark-s);
@include hljs-override('attr', $solarized-dark-na);
@include hljs-override('keyword', $solarized-dark-k);
@include hljs-override('variable', $solarized-dark-nv);
@include hljs-override('variable.language_', $solarized-dark-k);
@include hljs-override('title', $solarized-dark-nf);
@include hljs-override('name', $solarized-dark-k);
@include hljs-override('tag', $solarized-dark-nt);
@include hljs-override('type', $solarized-dark-nc);
@include hljs-override('number', $solarized-dark-mf);
@include hljs-override('literal', $solarized-dark-kc);
@include hljs-override('built_in', $solarized-dark-n);
@include hljs-override('section', $solarized-dark-gh);
@include hljs-override('bullet', $solarized-dark-n);
@include hljs-override('subst', $solarized-dark-p);
@include hljs-override('symbol', $solarized-dark-ni);
// Line numbers
.file-line-num {
@include line-number-link($solarized-dark-line-color);

8
app/views/groups/edit.html.haml
View File

@ -9,7 +9,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' }
= _('Naming, visibility')
%button.btn.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= _('Collapse')
%p
= _('Update your group name, description, avatar, and visibility.')
@ -21,7 +21,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' }
= _('Permissions and group features')
%button.btn.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= _('Configure advanced permissions, Large File Storage, two-factor authentication, and customer relations settings.')
@ -35,7 +35,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' }
= s_('GroupSettings|Badges')
%button.btn.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= s_('GroupSettings|Customize this group\'s badges.')
@ -51,7 +51,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only{ role: 'button' }
= _('Advanced')
%button.btn.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= _('Perform advanced options such as changing path, transferring, exporting, or removing the group.')

2
app/views/groups/merge_requests.html.haml
View File

@ -10,7 +10,7 @@
- if current_user
.nav-controls
- if @can_bulk_update
= render_if_exists 'shared/issuable/bulk_update_button', type: :merge_requests
= render_if_exists 'projects/merge_requests/bulk_update_button'
= render 'shared/new_project_item_select', path: 'merge_requests/new', label: _("merge request"), type: :merge_requests, with_feature_enabled: 'merge_requests', with_shared: false, include_projects_in_subgroups: true

2
app/views/groups/settings/_advanced.html.haml
View File

@ -4,7 +4,7 @@
.sub-section
%h4.warning-title= s_('GroupSettings|Change group URL')
= form_for @group, html: { multipart: true, class: 'gl-show-field-errors' }, authenticity_token: true do |f|
= form_errors(@group)
= form_errors(@group, pajamas_alert: true)
.form-group
%p
= s_("GroupSettings|Changing a group's URL can have unintended side effects.")

6
app/views/groups/settings/ci_cd/show.html.haml
View File

@ -11,7 +11,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only
= _("General pipelines")
%button.btn.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= _("Customize your pipeline configuration.")
@ -28,7 +28,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only
= _('Runners')
%button.btn.gl-button.btn-default.js-settings-toggle{ type: "button" }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= _("Runners are processes that pick up and execute CI/CD jobs for GitLab.")
@ -40,7 +40,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only
= _('Auto DevOps')
%button.btn.gl-button.btn-default.js-settings-toggle{ type: "button" }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
- auto_devops_url = help_page_path('topics/autodevops/index')

2
app/views/groups/settings/repository/_default_branch.html.haml
View File

@ -2,7 +2,7 @@
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only
= _('Default branch')
%button.gl-button.js-settings-toggle{ type: 'button' }
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded_by_default? ? _('Collapse') : _('Expand')
%p
= s_('GroupSettings|Set the initial name and protections for the default branch of new repositories created in the group.')

1
app/views/groups/settings/repository/show.html.haml
View File

@ -1,5 +1,6 @@
- breadcrumb_title _('Repository Settings')
- page_title _('Repository')
- @content_class = "limit-container-width" unless fluid_layout
- deploy_token_description = s_('DeployTokens|Group deploy tokens allow access to the packages, repositories, and registry images within the group.')

3
app/views/projects/branches/new.html.haml
View File

@ -26,7 +26,8 @@
= render 'shared/ref_dropdown', dropdown_class: 'wide'
.form-text.text-muted Existing branch name, tag, or commit SHA
.form-actions
= button_tag 'Create branch', class: 'gl-button btn btn-confirm'
= render Pajamas::ButtonComponent.new(variant: :confirm, button_options: { type: 'submit', class: 'gl-mr-3' }) do
= _('Create branch')
= link_to _('Cancel'), project_branches_path(@project), class: 'gl-button btn btn-default btn-cancel'
-# haml-lint:disable InlineJavaScript
%script#availableRefs{ type: "application/json" }= @project.repository.ref_names.to_json.html_safe

2
app/views/projects/issuable/_show.html.haml
View File

@ -4,7 +4,7 @@
- if issuable.relocation_target
- page_canonical_link issuable.relocation_target.present(current_user: current_user).web_url
= render "projects/issues/alert_moved_from_service_desk", issue: issuable
= render "projects/issues/service_desk/alert_moved_from_service_desk", issue: issuable
= render 'shared/issue_type/details_header', issuable: issuable
= render 'shared/issue_type/details_content', issuable: issuable, api_awards_path: api_awards_path

6
app/views/projects/issues/_by_email_description.html.haml
View File

@ -1,6 +0,0 @@
The subject will be used as the title of the new issue, and the message will be the description.
= link_to 'Quick actions', help_page_path('user/project/quick_actions'), target: '_blank', rel: 'noopener noreferrer'
and styling with
= link_to 'Markdown', help_page_path('user/markdown'), target: '_blank', rel: 'noopener noreferrer'
are supported.

8
app/views/projects/issues/service_desk.html.haml
View File

@ -3,7 +3,7 @@
- page_title _("Service Desk")
- add_page_specific_style 'page_bundles/issues_list'
- content_for :breadcrumbs_extra do
= render "projects/issues/nav_btns", show_export_button: false, show_rss_button: false
= render "projects/issues/service_desk/nav_btns", show_export_button: false, show_rss_button: false
- support_bot_attrs = { service_desk_enabled: @project.service_desk_enabled?, **UserSerializer.new.represent(User.support_bot) }.to_json
@ -11,12 +11,12 @@
.top-area
= render 'shared/issuable/nav', type: :issues
.nav-controls.d-block.d-sm-none
= render "projects/issues/nav_btns", show_feed_buttons: false, show_import_button: false, show_export_button: false
= render "projects/issues/service_desk/nav_btns", show_feed_buttons: false, show_import_button: false, show_export_button: false
- if @issues.present?
= render 'shared/issuable/search_bar', type: :issues
- if Gitlab::ServiceDesk.supported?
= render 'service_desk_info_content'
= render 'projects/issues/service_desk/service_desk_info_content'
.issues-holder
= render 'projects/issues/issues', empty_state_path: 'service_desk_empty_state'
= render 'projects/issues/issues', empty_state_path: 'projects/issues/service_desk/service_desk_empty_state'

0
app/views/projects/issues/_alert_moved_from_service_desk.html.haml → app/views/projects/issues/service_desk/_alert_moved_from_service_desk.html.haml
View File

0
app/views/projects/issues/_nav_btns.html.haml → app/views/projects/issues/service_desk/_nav_btns.html.haml
View File

0
app/views/projects/issues/_service_desk_empty_state.html.haml → app/views/projects/issues/service_desk/_service_desk_empty_state.html.haml
View File

0
app/views/projects/issues/_service_desk_info_content.html.haml → app/views/projects/issues/service_desk/_service_desk_info_content.html.haml
View File

4
app/views/shared/deploy_tokens/_index.html.haml
View File

@ -3,8 +3,8 @@
%section.settings.no-animate#js-deploy-tokens{ class: ('expanded' if expanded), data: { qa_selector: 'deploy_tokens_settings_content' } }
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only= s_('DeployTokens|Deploy tokens')
%button.btn.gl-button.btn-default.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
= render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do
= expanded ? _('Collapse') : _('Expand')
%p
= description
.settings-content

8
config/feature_flags/development/vsa_consistency_worker.yml
View File

@ -1,8 +0,0 @@
---
name: vsa_consistency_worker
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82591
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355709
milestone: '14.9'
type: development
group: group::optimize
default_enabled: true

6
config/initializers/truncato.rb Normal file
View File

@ -0,0 +1,6 @@
# frozen_string_literal: true
# Source: https://github.com/jorgemanrubia/truncato/issues/20#issuecomment-1135105823
silence_warnings do
Truncato::ARTIFICIAL_ROOT_NAME = 'truncato-artificial-root'
end

23
db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb Normal file
View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
class QueueBackfillProjectFeaturePackageRegistryAccessLevel < Gitlab::Database::Migration[2.0]
disable_ddl_transaction!
MIGRATION = 'BackfillProjectFeaturePackageRegistryAccessLevel'
DELAY_INTERVAL = 2.minutes
restrict_gitlab_migration gitlab_schema: :gitlab_main
def up
queue_batched_background_migration(
MIGRATION,
:projects,
:id,
job_interval: DELAY_INTERVAL
)
end
def down
delete_batched_background_migration(MIGRATION, :projects, :id, [])
end
end

1
db/schema_migrations/20220602070502 Normal file
View File

@ -0,0 +1 @@
9c66d020895c534280862136a08477fe3715465bdeb9d3e7dd632005c19de474

2
doc/administration/geo/replication/datatypes.md
View File

@ -206,7 +206,7 @@ successfully, you must replicate their data using some other means.
|[Incident Metric Images](../../../operations/incident_management/incidents.md#metrics) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | |
|[Alert Metric Images](../../../operations/incident_management/alerts.md#metrics-tab) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | |
|[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. |
|[Elasticsearch integration](../../../integration/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. |
|[Elasticsearch integration](../../../integration/advanced_search/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. |
|[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. |
|[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. |

2
doc/administration/gitaly/configure_gitaly.md
View File

@ -67,7 +67,7 @@ When running Gitaly on its own server, note the following regarding GitLab versi
servers.
- From GitLab 11.8 to 12.2, it is possible to use Elasticsearch in a Gitaly setup that doesn't use
NFS. To use Elasticsearch in these versions, the
[repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer)
[repository indexer](../../integration/advanced_search/elasticsearch.md#elasticsearch-repository-indexer)
must be enabled in your GitLab configuration.
- [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is
the default and no configuration is required.

2
doc/administration/index.md
View File

@ -65,7 +65,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
requests diffs external storage.
- [Broadcast Messages](../user/admin_area/broadcast_messages.md): Send messages
to GitLab users through the UI.
- [Elasticsearch](../integration/elasticsearch.md): Enable Elasticsearch to
- [Elasticsearch](../integration/advanced_search/elasticsearch.md): Enable Elasticsearch to
empower Advanced Search. Use when you deal with a huge amount of data.
- [External Classification Policy Authorization](../user/admin_area/settings/external_authorization.md)
- [Add a license](../user/admin_area/license.md): Add a license to unlock

2
doc/administration/instance_limits.md
View File

@ -924,7 +924,7 @@ indexed, which have a separate limit. For more information, read
- For self-managed installations, the field length is unlimited by default.
You can configure this limit for self-managed installations when you
[enable Elasticsearch](../integration/elasticsearch.md#enable-advanced-search).
[enable Elasticsearch](../integration/advanced_search/elasticsearch.md#enable-advanced-search).
Set the limit to `0` to disable it.
## Wiki limits

4
doc/administration/reference_architectures/10k_users.md
View File

@ -2233,13 +2233,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t
## Configure Advanced Search
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

4
doc/administration/reference_architectures/1k_users.md
View File

@ -106,13 +106,13 @@ performance and reliability at an increased complexity cost.
## Configure Advanced Search **(PREMIUM SELF)**
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
## Cloud Native Hybrid reference architecture with Helm Charts

4
doc/administration/reference_architectures/25k_users.md
View File

@ -2237,13 +2237,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t
## Configure Advanced Search
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

4
doc/administration/reference_architectures/2k_users.md
View File

@ -941,13 +941,13 @@ in the future.
## Configure Advanced Search **(PREMIUM SELF)**
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

4
doc/administration/reference_architectures/3k_users.md
View File

@ -2172,13 +2172,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t
## Configure Advanced Search
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

4
doc/administration/reference_architectures/50k_users.md
View File

@ -2253,13 +2253,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t
## Configure Advanced Search
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

4
doc/administration/reference_architectures/5k_users.md
View File

@ -2172,13 +2172,13 @@ While sharing the job logs through NFS is supported, it's recommended to avoid t
## Configure Advanced Search
You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md)
You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md)
for faster, more advanced code search across your entire GitLab instance.
Elasticsearch cluster design and requirements are dependent on your specific
data. For recommended best practices about how to set up your Elasticsearch
cluster alongside your instance, read how to
[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration).
<div align="right">
<a type="button" class="btn btn-default" href="#setup-components">

13
doc/administration/troubleshooting/elasticsearch.md
View File

@ -6,8 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Troubleshooting Elasticsearch **(PREMIUM SELF)**
To install and configure Elasticsearch, and for common and known issues,
visit the [administrator documentation](../../integration/elasticsearch.md).
To install and configure Elasticsearch,
visit the [administrator documentation](../../integration/advanced_search/elasticsearch.md).
For troubleshooting, visit the
[administrator troubleshooting documentation](../../integration/advanced_search/elasticsearch_troubleshooting.md).
Troubleshooting Elasticsearch requires:
@ -242,7 +245,7 @@ The best place to start is to determine if the issue is with creating an empty i
If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the
name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch
side and attempt to recreate it from the
[`recreate_index`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks)
[`recreate_index`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks)
Rake task.
If you still encounter issues, try creating an index manually on the Elasticsearch
@ -261,8 +264,8 @@ during the indexing of projects. If errors do occur, they stem from either the i
If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks:
- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status)
- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed)
- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status)
- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed)
If:

4
doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
View File

@ -1446,7 +1446,7 @@ Open the rails console (`gitlab rails c`) and run the following command to see a
ApplicationSetting.last.attributes
```
Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`.
Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`.
#### Setting attributes
@ -1462,7 +1462,7 @@ ApplicationSetting.last.update(elasticsearch_indexing: false)
#### Getting attributes
You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/elasticsearch.md) or in the rails console by issuing:
You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md) or in the rails console by issuing:
```ruby
Gitlab::CurrentSettings.elasticsearch_url

28
doc/api/search.md
View File

@ -29,7 +29,7 @@ GET /search
Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `snippet_titles`, `users`.
If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)**
If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)**
The response depends on the requested scope.
@ -270,7 +270,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye"
@ -301,7 +301,7 @@ NOTE:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=commits&search=bye"
@ -336,7 +336,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
Filters are available for this scope:
@ -377,7 +377,7 @@ NOTE:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime"
@ -452,7 +452,7 @@ GET /groups/:id/search
Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `users`.
If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)**
If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)**
The response depends on the requested scope.
@ -662,7 +662,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=wiki_blobs&search=bye"
@ -693,7 +693,7 @@ NOTE:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=commits&search=bye"
@ -728,7 +728,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
Filters are available for this scope:
@ -769,7 +769,7 @@ NOTE:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime"
@ -1021,7 +1021,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime"
@ -1057,7 +1057,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
Filters are available for this scope:
@ -1105,7 +1105,7 @@ NOTE:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=commits&search=bye"
@ -1140,7 +1140,7 @@ Example response:
> Moved to GitLab Premium in 13.9.
This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled.
Filters are available for this scope:

2
doc/development/adding_service_component.md
View File

@ -47,7 +47,7 @@ Adding a new service follows the same [merge request workflow](contributing/merg
The first iteration should be to add the ability to connect and use the service as an externally installed component. Often this involves providing settings in GitLab to connect to the service, or allow connections from it. And then shipping documentation on how to install and configure the service with GitLab.
[Elasticsearch](../integration/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives.
[Elasticsearch](../integration/advanced_search/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives.
**For services that depend on the existing GitLab codebase:**

6
doc/development/architecture.md
View File

@ -442,9 +442,9 @@ Consul is a tool for service discovery and configuration. Consul is distributed,
- [Project page](https://github.com/elastic/elasticsearch/)
- Configuration:
- [Omnibus](../integration/elasticsearch.md)
- [Charts](../integration/elasticsearch.md)
- [Source](../integration/elasticsearch.md)
- [Omnibus](../integration/advanced_search/elasticsearch.md)
- [Charts](../integration/advanced_search/elasticsearch.md)
- [Source](../integration/advanced_search/elasticsearch.md)
- [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md)
- Layer: Core Service (Data)
- GitLab.com: [Get Advanced Search working on GitLab.com (Closed)](https://gitlab.com/groups/gitlab-org/-/epics/153) epic.

2
doc/development/database/batched_background_migrations.md
View File

@ -261,7 +261,7 @@ background migration.
end
def down
delete_batched_background_migration(MIGRATION_NAME, :routes, :id, [])
delete_batched_background_migration(MIGRATION, :routes, :id, [])
end
end
```

6
doc/development/elasticsearch.md
View File

@ -9,17 +9,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w
This area is to maintain a compendium of useful information when working with Elasticsearch.
Information on how to enable Elasticsearch and perform the initial indexing is in
the [Elasticsearch integration documentation](../integration/elasticsearch.md#enable-advanced-search).
the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search).
## Deep Dive
In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction.
In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction.
In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3.
## Supported Versions
See [Version Requirements](../integration/elasticsearch.md#version-requirements).
See [Version Requirements](../integration/advanced_search/elasticsearch.md#version-requirements).
Developers making significant changes to Elasticsearch queries should test their features against all our supported versions.

2
doc/index.md
View File

@ -50,7 +50,7 @@ Have a look at some of our most popular topics:
| [Activate GitLab EE with a license](user/admin_area/license.md) | Activate GitLab Enterprise Edition functionality with a license. |
| [Back up and restore GitLab](raketasks/backup_restore.md) | Rake tasks for backing up and restoring GitLab self-managed instances. |
| [GitLab release and maintenance policy](policy/maintenance.md) | Policies for version naming and cadence, and also upgrade recommendations. |
| [Elasticsearch integration](integration/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. |
| [Elasticsearch integration](integration/advanced_search/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. |
| [Omnibus GitLab database settings](https://docs.gitlab.com/omnibus/settings/database.html) | Database settings for Omnibus GitLab self-managed instances. |
| [Omnibus GitLab NGINX settings](https://docs.gitlab.com/omnibus/settings/nginx.html) | NGINX settings for Omnibus GitLab self-managed instances. |
| [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl.html) | SSL settings for Omnibus GitLab self-managed instances. |

2
doc/install/next_steps.md
View File

@ -56,7 +56,7 @@ installation.
## Cross-repository Code Search
- [Advanced Search](../integration/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for
- [Advanced Search](../integration/advanced_search/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for
faster, more advanced code search across your entire GitLab instance.
## Scaling and replication

857
doc/integration/advanced_search/elasticsearch.md Normal file
View File

@ -0,0 +1,857 @@
---
type: reference
stage: Data Stores
group: Global Search
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Elasticsearch integration **(PREMIUM SELF)**
This page describes how to enable Advanced Search. When enabled,
Advanced Search provides faster search response times and [improved search features](../../user/search/advanced_search.md).
## Version requirements
### Elasticsearch version requirements
> Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0.
Advanced Search works with the following versions of Elasticsearch.
| GitLab version | Elasticsearch version |
|-----------------------|--------------------------|
| GitLab 15.0 or later | Elasticsearch 7.x - 8.x |
| GitLab 13.9 - 14.10 | Elasticsearch 6.8 - 7.x |
| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x |
| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x |
Advanced Search follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol).
When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts
before we remove them.
### OpenSearch version requirements
| GitLab version | Elasticsearch version |
|-----------------------|--------------------------|
| GitLab 15.0 or later | OpenSearch 1.x or later |
If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing).
## System requirements
Elasticsearch requires additional resources to those documented in the
[GitLab system requirements](../../install/requirements.md).
Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to
[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory),
each node should have:
- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum).
- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs.
- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements.
## Install Elasticsearch
Elasticsearch is *not* included in the Omnibus packages or when you install from
source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page.
You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html)
service.
You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance.
For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards.
The search index updates after you:
- Add data to the database or repository.
- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area.
## Upgrade to a new Elasticsearch major version
> - Elasticsearch 6.8 support is removed with GitLab 15.0.
> - Upgrading from GitLab 14.10 to 15.0 requires that you are using any version of Elasticsearch 7.x.
You are not required to change the GitLab configuration when you upgrade Elasticsearch.
## Elasticsearch repository indexer
To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer).
Depending on your GitLab version, there are different installation procedures for the Go indexer:
- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab).
- For installations from source or older versions of Omnibus GitLab,
[install the indexer from source](#from-source).
- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md).
### Omnibus GitLab
Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab.
The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481).
### From source
First, we need to install some dependencies, then we build and install
the indexer itself.
This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding,
therefore we must ensure the development packages for your platform are
installed before running `make`.
#### Debian / Ubuntu
To install on Debian or Ubuntu, run:
```shell
sudo apt install libicu-dev
```
#### CentOS / RHEL
To install on CentOS or RHEL, run:
```shell
sudo yum install libicu-devel
```
#### macOS
NOTE:
You must first [install Homebrew](https://brew.sh/).
To install on macOS, run:
```shell
brew install icu4c
export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
```
### Build and install
To build and install the indexer, run:
```shell
indexer_path=/home/git/gitlab-elasticsearch-indexer
# Run the installation task for gitlab-elasticsearch-indexer:
sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production
cd $indexer_path && sudo make install
```
The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`.
You can change the installation path with the `PREFIX` environment variable.
Please remember to pass the `-E` flag to `sudo` if you do so.
Example:
```shell
PREFIX=/usr sudo -E make install
```
After installation, be sure to [enable Elasticsearch](#enable-advanced-search).
NOTE:
If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you
may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to
`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed.
## Enable Advanced Search
For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below.
To enable Advanced Search, you must have administrator access to GitLab:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
NOTE:
To see the Advanced Search section, you need an active GitLab Premium
[license](../../user/admin_area/license.md).
1. Configure the [Advanced Search settings](#advanced-search-configuration) for
your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled**
yet.
1. Enable **Elasticsearch indexing** and select **Save changes**. This creates
an empty index if one does not already exist.
1. Select **Index all projects**.
1. Select **Check progress** in the confirmation message to see the status of
the background jobs.
1. Personal snippets must be indexed using another Rake task:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_snippets
# Installations from source
bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production
```
1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**.
NOTE:
When your Elasticsearch cluster is down while Elasticsearch is enabled,
you might have problems updating documents such as issues because your
instance queues a job to index the change, but cannot find a valid
Elasticsearch cluster.
### Advanced Search configuration
The following Elasticsearch settings are available:
| Parameter | Description |
|-------------------------------------------------------|-------------|
| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. |
| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. |
| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. |
| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. |
| `Username` | The `username` of your Elasticsearch instance. |
| `Password` | The password of your Elasticsearch instance. |
| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). |
| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. |
| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).
| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. |
| `AWS Region` | The AWS region in which your OpenSearch Service is located. |
| `AWS Access Key` | The AWS access key. |
| `AWS Secret Access Key` | The AWS secret access key. |
| `Maximum file size indexed` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-file-size-indexed). |
| `Maximum field length` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-field-length). |
| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. |
| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. |
| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. |
WARNING:
Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact
Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations
in your Sidekiq logs. For more information, see
[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147).
### Limit the number of namespaces and projects that can be indexed
If you check checkbox `Limit the number of namespaces and projects that can be indexed`
under **Elasticsearch indexing restrictions** more options become available.
![limit namespaces and projects options](img/limit_namespaces_projects_options.png)
You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes
any subgroups and projects belonging to those subgroups to be indexed as well.
Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second.
You can filter the selection dropdown by writing part of the namespace or project name you're interested in.
![limit namespace filter](img/limit_namespace_filter.png)
NOTE:
If no namespaces or projects are selected, no Advanced Search indexing takes place.
WARNING:
If you have already indexed your instance, you must regenerate the index to delete all existing data
for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and
`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data
from the Elasticsearch index as expected.
## Enable custom language analyzers
You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic.
To enable languages support:
1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Locate **Custom analyzers: language support**.
1. Enable plugins support for **Indexing**.
1. Select **Save changes** for the changes to take effect.
1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings.
1. Enable plugins support for **Searching** after the previous step is completed.
For guidance on what to install, see the following Elasticsearch language plugin options:
| Parameter | Description |
|-------------------------------------------------------|-------------|
| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.|
| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.|
| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.|
| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.|
## Disable Advanced Search
To disable the Elasticsearch integration:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**.
1. Select **Save changes**.
1. Optional. Delete the existing indices:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:delete_index
# Installations from source
bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production
```
## Unpause Indexing
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Advanced Search**.
1. Clear the **Pause Elasticsearch indexing** checkbox.
## Zero downtime reindexing
The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html)
and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a
`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause
the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the
index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the
index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes.
### Trigger the reindex via the Advanced Search administration
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2.
> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3.
> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12.
To trigger the reindexing process:
1. Sign in to your GitLab instance as an administrator.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Elasticsearch zero-downtime reindexing**.
1. Select **Trigger cluster reindexing**.
Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster.
After this process is completed, the original index is scheduled to be deleted after
14 days. You can cancel this action by pressing the **Cancel** button on the same
page you triggered the reindexing process.
While the reindexing is running, you can follow its progress under that same section.
#### Elasticsearch zero-downtime reindexing
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Elasticsearch zero-downtime reindexing**, and you'll
find the following options:
- [Slice multiplier](#slice-multiplier)
- [Maximum running slices](#maximum-running-slices)
##### Slice multiplier
The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice).
GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice)
to control the reindex efficiently and safely, which enables users to retry only
failed slices.
The multiplier defaults to `2` and applies to the number of shards per index.
For example, if this value is `2` and your index has 20 shards, then the
reindex task is split into 40 slices.
##### Maximum running slices
The maximum running slices parameter defaults to `60` and corresponds to the
maximum number of slices allowed to run concurrently during Elasticsearch
reindexing.
Setting this value too high can have adverse performance impacts as your cluster
may become heavily saturated with searches and writes. Setting this value too
low may lead the reindexing process to take a very long time to complete.
The best value for this depends on your cluster size, whether you're willing
to accept some degraded search performance during reindexing, and how important
it is for the reindex to finish quickly and resume indexing.
### Mark the most recent reindex job as failed and resume the indexing
Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps:
1. Mark the most recent reindex job as failed:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:mark_reindex_failed
# Installations from source
bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production
```
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Advanced Search**.
1. Clear the **Pause Elasticsearch indexing** checkbox.
## Advanced Search migrations
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6.
With reindex migrations running in the background, there's no need for a manual
intervention. This usually happens in situations where new features are added to
Advanced Search, which means adding or changing the way content is indexed.
To confirm that the Advanced Search migrations ran, you can check with:
```shell
curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq .
```
This should return something similar to:
```json
{
"took": 14,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "gitlab-production-migrations",
"_type": "_doc",
"_id": "20201105181100",
"_score": 1,
"_source": {
"completed": true
}
}
]
}
}
```
In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog).
### Retry a halted migration
Some migrations are built with a retry limit. If the migration cannot finish within the retry limit,
it is halted and a notification is displayed in the Advanced Search integration settings.
It is recommended to check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog) to
debug why the migration was halted and make any changes before retrying the migration. Once you believe you've
fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried
in the background.
If you cannot get the migration to succeed, you may
consider the
[last resort to recreate the index from scratch](elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index).
This may allow you to skip over
the problem because a newly created index skips all migrations as the index
is recreated with the correct up-to-date schema.
### All migrations must be finished before doing a major upgrade
Before doing a major version GitLab upgrade, you should have completed all
migrations that exist up until the latest minor version before that major
version. If you have halted migrations, these need to be resolved and
[retried](#retry-a-halted-migration) before proceeding with a major version
upgrade. Read more about [upgrading to a new major
version](../../update/index.md#upgrading-to-a-new-major-version).
## GitLab Advanced Search Rake tasks
Rake tasks are available to:
- [Build and install](#build-and-install) the indexer.
- Delete indices when [disabling Elasticsearch](#disable-advanced-search).
- Add GitLab data to an index.
The following are some available Rake tasks:
| Task | Description |
|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. |
| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. |
| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. |
| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. |
| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. |
| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. |
| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indices (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. |
| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indices and aliases (if they exist) on the Elasticsearch instance. |
| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. |
| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. |
| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. |
| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. |
| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. |
| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. |
| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. |
| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. |
| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. |
### Environment variables
In addition to the Rake tasks, there are some environment variables that can be used to modify the process:
| Environment Variable | Data Type | What it does |
| -------------------- |:---------:| ---------------------------------------------------------------------------- |
| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). |
| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. |
| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. |
### Indexing a range of projects or a specific project
Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing.
```shell
root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100
```
Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project
by setting both to the same project ID:
```shell
root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5
Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)...
I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done!
```
## Advanced Search index scopes
When performing a search, the GitLab index uses the following scopes:
| Scope Name | What it searches |
| ---------------- | ---------------------- |
| `commits` | Commit data |
| `projects` | Project data (default) |
| `blobs` | Code |
| `issues` | Issue data |
| `merge_requests` | Merge request data |
| `milestones` | Milestone data |
| `notes` | Note data |
| `snippets` | Snippet data |
| `wiki_blobs` | Wiki contents |
## Tuning
### Guidance on choosing optimal cluster configuration
For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below.
- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand.
- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example).
- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations.
- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details.
- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below.
- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health.
- Number of Elasticsearch shards:
- Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB.
- Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example:
- If you have fewer than about 2,000,000 documents, use the default of 5 shards
- 10,000,000 documents: `10000000/5000000 + 5` = 7 shards
- 100,000,000 documents: `100000000/5000000 + 5` = 25 shards
- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value.
- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations.
### Advanced Search integration settings guidance
- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards.
- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index.
### How to index large instances efficiently
This section may be helpful in the event that the other
[basic instructions](#enable-advanced-search) cause problems
due to large volumes of data being indexed.
WARNING:
Indexing a large instance generates a lot of Sidekiq jobs.
Make sure to prepare for this task by having a [Scalable and Highly Available
Setup](../../administration/reference_architectures/index.md) or creating [extra
Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md).
1. [Configure your Elasticsearch host and port](#enable-advanced-search).
1. Create empty indices:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:create_empty_index
# Installations from source
bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production
```
1. If this is a re-index of your GitLab instance, clear the index status:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:clear_index_status
# Installations from source
bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production
```
1. [Enable **Elasticsearch indexing**](#enable-advanced-search).
1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed):
- You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search.
- You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete.
In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings.
NOTE:
This step is optional but may help significantly speed up large indexing operations.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"index" : {
"refresh_interval" : "-1",
"number_of_replicas" : 0
} }'
```
1. Index projects and their associated data:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects
# Installations from source
bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production
```
This enqueues a Sidekiq job for each project that needs to be indexed.
You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab**
and select `elastic_commit_indexer`, or you can query indexing status using a Rake task:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects_status
# Installations from source
bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production
Indexing is 65.55% complete (6555/10000 projects)
```
If you want to limit the index to a range of projects you can provide the
`ID_FROM` and `ID_TO` parameters:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000
# Installations from source
bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production
```
Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional.
The above example will index all projects from ID `1001` up to (and including) ID `2000`.
NOTE:
Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects`
can get interrupted. This may happen for many reasons, but it's always safe
to run the indexing task again. It will skip repositories that have
already been indexed.
As the indexer stores the last commit SHA of every indexed repository in the
database, you can run the indexer with the special parameter `UPDATE_INDEX` and
it will check every project repository again to make sure that every commit in
a repository is indexed, which can be useful in case if your index is outdated:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000
# Installations from source
bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production
```
You can also use the `gitlab:elastic:clear_index_status` Rake task to force the
indexer to "forget" all progress, so it retries the indexing process from the
start.
1. Personal snippets are not associated with a project and need to be indexed separately:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_snippets
# Installations from source
bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production
```
1. Enable replication and refreshing again after indexing (only if you previously disabled it):
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"index" : {
"number_of_replicas" : 1,
"refresh_interval" : "1s"
} }'
```
A force merge should be called after enabling the refreshing above.
For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge:
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"settings": {
"index.blocks.write": true
} }'
```
Then, initiate the force merge:
```shell
curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5'
```
After this, if your index is in read-only mode, switch back to read-write:
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"settings": {
"index.blocks.write": false
} }'
```
1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search).
### Deleted documents
Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index.
In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_
However, some larger installations may wish to tune the merge policy settings:
- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \
--data '{
"index" : {
"merge.policy.max_merged_segment": "2gb"
}
}'
```
- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \
--data '{
"index" : {
"merge.policy.reclaim_deletes_weight": "3.0"
}
}'
```
- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues.
## Index large instances with dedicated Sidekiq nodes or processes
Indexing a large instance can be a lengthy and resource-intensive process that has the potential
of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and
availability.
As GitLab allows you to start multiple Sidekiq processes, you can create an
additional process dedicated to indexing a set of queues (or queue group). This way, you can
ensure that indexing queues always have a dedicated worker, while the rest of the queues have
another dedicated worker to avoid contention.
For this purpose, use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector)
option that allows a more general selection of queue groups using a [worker matching query](../../administration/operations/extra_sidekiq_routing.md#worker-matching-query).
To handle these two queue groups, we generally recommend one of the following two options. You can either:
- [Use two queue groups on one single node](#single-node-two-processes).
- [Use two queue groups, one on each node](#two-nodes-one-process-for-each).
For the steps below, consider:
- `feature_category=global_search` as an indexing queue group with its own Sidekiq process.
- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process.
### Single node, two processes
To create both an indexing and a non-indexing Sidekiq process in one node:
1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category=global_search",
"feature_category!=global_search"
]
```
1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md)
for the changes to take effect.
WARNING:
When starting multiple processes, the number of processes cannot exceed the number of CPU
cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject
to the available workload and concurrency settings. For more details, see how to
[run multiple Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md).
### Two nodes, one process for each
To handle these queue groups on two nodes:
1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category=global_search"
]
```
1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md)
for the changes to take effect.
1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category!=global_search"
]
```
to set up a non-indexing Sidekiq process.
1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md)
for the changes to take effect.
## Reverting to Basic Search
Sometimes there may be issues with your Elasticsearch index data and as such
GitLab allows you to revert to "basic search" when there are no search
results and assuming that basic search is supported in that scope. This "basic
search" behaves as though you don't have Advanced Search enabled at all for
your instance and search using other data sources (such as PostgreSQL data and Git
data).
## Data recovery: Elasticsearch is a secondary data store only
The use of Elasticsearch in GitLab is only ever as a secondary data store.
This means that all of the data stored in Elasticsearch can always be derived
again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if
the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch.

18
doc/integration/advanced_search/elasticsearch_troubleshooting.md
View File

@ -57,11 +57,11 @@ There are a couple of ways to achieve that:
We continuously make updates to our indexing strategies and aim to support
newer versions of Elasticsearch. When indexing changes are made, it may
be necessary for you to [reindex](../elasticsearch.md#zero-downtime-reindexing) after updating GitLab.
be necessary for you to [reindex](elasticsearch.md#zero-downtime-reindexing) after updating GitLab.
## I indexed all the repositories but I can't get any hits for my search term in the UI
Make sure you [indexed all the database data](../elasticsearch.md#enable-advanced-search).
Make sure you [indexed all the database data](elasticsearch.md#enable-advanced-search).
If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`):
@ -82,9 +82,9 @@ More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticse
It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../../administration/troubleshooting/elasticsearch.md#search-results-workflow).
NOTE:
The above instructions are not to be used for scenarios that only index a [subset of namespaces](../elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).
The above instructions are not to be used for scenarios that only index a [subset of namespaces](elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).
See [Elasticsearch Index Scopes](../elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data.
See [Elasticsearch Index Scopes](elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data.
## I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything
@ -133,7 +133,7 @@ see details in the [update guide](../../update/upgrading_from_source.md).
## `Elasticsearch::Transport::Transport::Errors::BadRequest`
If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](../elasticsearch.md#system-requirements).
If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](elasticsearch.md#system-requirements).
There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command.
## `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge`
@ -173,7 +173,7 @@ Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="he
```
You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a).
After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](../elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](../elasticsearch.md#enable-advanced-search).
After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](elasticsearch.md#enable-advanced-search).
## My Elasticsearch cluster has a plugin and the integration is not working
@ -239,13 +239,13 @@ filtering out projects that a user does not have access to at search time.
If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings:
- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](../elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations.
- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](../elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests.
- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations.
- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests.
## Indexing is very slow or fails with `rejected execution of coordinating operation` messages
Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory.
Ensure that your Elasticsearch cluster meets the [system requirements](../elasticsearch.md#system-requirements) and has enough resources
Ensure that your Elasticsearch cluster meets the [system requirements](elasticsearch.md#system-requirements) and has enough resources
to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests).
## Access requirements for the self-managed AWS OpenSearch Service

0
doc/integration/img/limit_namespace_filter.png → doc/integration/advanced_search/img/limit_namespace_filter.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.4 KiB

0
doc/integration/img/limit_namespaces_projects_options.png → doc/integration/advanced_search/img/limit_namespaces_projects_options.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

860
doc/integration/elasticsearch.md
View File

@ -1,857 +1,11 @@
---
type: reference
stage: Data Stores
group: Global Search
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
redirect_to: 'advanced_search/elasticsearch.md'
remove_date: '2022-09-13'
---
# Elasticsearch integration **(PREMIUM SELF)**
This document was moved to [another location](advanced_search/elasticsearch.md).
This page describes how to enable Advanced Search. When enabled,
Advanced Search provides faster search response times and [improved search features](../user/search/advanced_search.md).
## Version requirements
### Elasticsearch version requirements
> Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0.
Advanced Search works with the following versions of Elasticsearch.
| GitLab version | Elasticsearch version |
|-----------------------|--------------------------|
| GitLab 15.0 or later | Elasticsearch 7.x - 8.x |
| GitLab 13.9 - 14.10 | Elasticsearch 6.8 - 7.x |
| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x |
| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x |
Advanced Search follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol).
When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts
before we remove them.
### OpenSearch version requirements
| GitLab version | Elasticsearch version |
|-----------------------|--------------------------|
| GitLab 15.0 or later | OpenSearch 1.x or later |
If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing).
## System requirements
Elasticsearch requires additional resources to those documented in the
[GitLab system requirements](../install/requirements.md).
Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to
[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory),
each node should have:
- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum).
- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs.
- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements.
## Install Elasticsearch
Elasticsearch is *not* included in the Omnibus packages or when you install from
source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page.
You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html)
service.
You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance.
For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards.
The search index updates after you:
- Add data to the database or repository.
- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area.
## Upgrade to a new Elasticsearch major version
> - Elasticsearch 6.8 support is removed with GitLab 15.0.
> - Upgrading from GitLab 14.10 to 15.0 requires that you are using any version of Elasticsearch 7.x.
You are not required to change the GitLab configuration when you upgrade Elasticsearch.
## Elasticsearch repository indexer
To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer).
Depending on your GitLab version, there are different installation procedures for the Go indexer:
- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab).
- For installations from source or older versions of Omnibus GitLab,
[install the indexer from source](#from-source).
- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md).
### Omnibus GitLab
Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab.
The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481).
### From source
First, we need to install some dependencies, then we build and install
the indexer itself.
This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding,
therefore we must ensure the development packages for your platform are
installed before running `make`.
#### Debian / Ubuntu
To install on Debian or Ubuntu, run:
```shell
sudo apt install libicu-dev
```
#### CentOS / RHEL
To install on CentOS or RHEL, run:
```shell
sudo yum install libicu-devel
```
#### macOS
NOTE:
You must first [install Homebrew](https://brew.sh/).
To install on macOS, run:
```shell
brew install icu4c
export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
```
### Build and install
To build and install the indexer, run:
```shell
indexer_path=/home/git/gitlab-elasticsearch-indexer
# Run the installation task for gitlab-elasticsearch-indexer:
sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production
cd $indexer_path && sudo make install
```
The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`.
You can change the installation path with the `PREFIX` environment variable.
Please remember to pass the `-E` flag to `sudo` if you do so.
Example:
```shell
PREFIX=/usr sudo -E make install
```
After installation, be sure to [enable Elasticsearch](#enable-advanced-search).
NOTE:
If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you
may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to
`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed.
## Enable Advanced Search
For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below.
To enable Advanced Search, you must have administrator access to GitLab:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
NOTE:
To see the Advanced Search section, you need an active GitLab Premium
[license](../user/admin_area/license.md).
1. Configure the [Advanced Search settings](#advanced-search-configuration) for
your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled**
yet.
1. Enable **Elasticsearch indexing** and select **Save changes**. This creates
an empty index if one does not already exist.
1. Select **Index all projects**.
1. Select **Check progress** in the confirmation message to see the status of
the background jobs.
1. Personal snippets must be indexed using another Rake task:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_snippets
# Installations from source
bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production
```
1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**.
NOTE:
When your Elasticsearch cluster is down while Elasticsearch is enabled,
you might have problems updating documents such as issues because your
instance queues a job to index the change, but cannot find a valid
Elasticsearch cluster.
### Advanced Search configuration
The following Elasticsearch settings are available:
| Parameter | Description |
|-------------------------------------------------------|-------------|
| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. |
| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. |
| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. |
| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. |
| `Username` | The `username` of your Elasticsearch instance. |
| `Password` | The password of your Elasticsearch instance. |
| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). |
| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. |
| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).
| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. |
| `AWS Region` | The AWS region in which your OpenSearch Service is located. |
| `AWS Access Key` | The AWS access key. |
| `AWS Secret Access Key` | The AWS secret access key. |
| `Maximum file size indexed` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-file-size-indexed). |
| `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). |
| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. |
| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. |
| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. |
WARNING:
Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact
Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations
in your Sidekiq logs. For more information, see
[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147).
### Limit the number of namespaces and projects that can be indexed
If you check checkbox `Limit the number of namespaces and projects that can be indexed`
under **Elasticsearch indexing restrictions** more options become available.
![limit namespaces and projects options](img/limit_namespaces_projects_options.png)
You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes
any subgroups and projects belonging to those subgroups to be indexed as well.
Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second.
You can filter the selection dropdown by writing part of the namespace or project name you're interested in.
![limit namespace filter](img/limit_namespace_filter.png)
NOTE:
If no namespaces or projects are selected, no Advanced Search indexing takes place.
WARNING:
If you have already indexed your instance, you must regenerate the index to delete all existing data
for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and
`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data
from the Elasticsearch index as expected.
## Enable custom language analyzers
You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic.
To enable languages support:
1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Locate **Custom analyzers: language support**.
1. Enable plugins support for **Indexing**.
1. Select **Save changes** for the changes to take effect.
1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings.
1. Enable plugins support for **Searching** after the previous step is completed.
For guidance on what to install, see the following Elasticsearch language plugin options:
| Parameter | Description |
|-------------------------------------------------------|-------------|
| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.|
| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.|
| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.|
| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.|
## Disable Advanced Search
To disable the Elasticsearch integration:
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**.
1. Select **Save changes**.
1. Optional. Delete the existing indices:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:delete_index
# Installations from source
bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production
```
## Unpause Indexing
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Advanced Search**.
1. Clear the **Pause Elasticsearch indexing** checkbox.
## Zero downtime reindexing
The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html)
and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a
`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause
the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the
index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the
index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes.
### Trigger the reindex via the Advanced Search administration
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2.
> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3.
> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12.
To trigger the reindexing process:
1. Sign in to your GitLab instance as an administrator.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Elasticsearch zero-downtime reindexing**.
1. Select **Trigger cluster reindexing**.
Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster.
After this process is completed, the original index is scheduled to be deleted after
14 days. You can cancel this action by pressing the **Cancel** button on the same
page you triggered the reindexing process.
While the reindexing is running, you can follow its progress under that same section.
#### Elasticsearch zero-downtime reindexing
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12.
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Elasticsearch zero-downtime reindexing**, and you'll
find the following options:
- [Slice multiplier](#slice-multiplier)
- [Maximum running slices](#maximum-running-slices)
##### Slice multiplier
The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice).
GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice)
to control the reindex efficiently and safely, which enables users to retry only
failed slices.
The multiplier defaults to `2` and applies to the number of shards per index.
For example, if this value is `2` and your index has 20 shards, then the
reindex task is split into 40 slices.
##### Maximum running slices
The maximum running slices parameter defaults to `60` and corresponds to the
maximum number of slices allowed to run concurrently during Elasticsearch
reindexing.
Setting this value too high can have adverse performance impacts as your cluster
may become heavily saturated with searches and writes. Setting this value too
low may lead the reindexing process to take a very long time to complete.
The best value for this depends on your cluster size, whether you're willing
to accept some degraded search performance during reindexing, and how important
it is for the reindex to finish quickly and resume indexing.
### Mark the most recent reindex job as failed and resume the indexing
Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps:
1. Mark the most recent reindex job as failed:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:mark_reindex_failed
# Installations from source
bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production
```
1. On the top bar, select **Menu > Admin**.
1. On the left sidebar, select **Settings > Advanced Search**.
1. Expand **Advanced Search**.
1. Clear the **Pause Elasticsearch indexing** checkbox.
## Advanced Search migrations
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6.
With reindex migrations running in the background, there's no need for a manual
intervention. This usually happens in situations where new features are added to
Advanced Search, which means adding or changing the way content is indexed.
To confirm that the Advanced Search migrations ran, you can check with:
```shell
curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq .
```
This should return something similar to:
```json
{
"took": 14,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "gitlab-production-migrations",
"_type": "_doc",
"_id": "20201105181100",
"_score": 1,
"_source": {
"completed": true
}
}
]
}
}
```
In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog).
### Retry a halted migration
Some migrations are built with a retry limit. If the migration cannot finish within the retry limit,
it is halted and a notification is displayed in the Advanced Search integration settings.
It is recommended to check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog) to
debug why the migration was halted and make any changes before retrying the migration. Once you believe you've
fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried
in the background.
If you cannot get the migration to succeed, you may
consider the
[last resort to recreate the index from scratch](advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index).
This may allow you to skip over
the problem because a newly created index skips all migrations as the index
is recreated with the correct up-to-date schema.
### All migrations must be finished before doing a major upgrade
Before doing a major version GitLab upgrade, you should have completed all
migrations that exist up until the latest minor version before that major
version. If you have halted migrations, these need to be resolved and
[retried](#retry-a-halted-migration) before proceeding with a major version
upgrade. Read more about [upgrading to a new major
version](../update/index.md#upgrading-to-a-new-major-version).
## GitLab Advanced Search Rake tasks
Rake tasks are available to:
- [Build and install](#build-and-install) the indexer.
- Delete indices when [disabling Elasticsearch](#disable-advanced-search).
- Add GitLab data to an index.
The following are some available Rake tasks:
| Task | Description |
|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. |
| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. |
| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. |
| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. |
| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. |
| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. |
| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indices (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. |
| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indices and aliases (if they exist) on the Elasticsearch instance. |
| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. |
| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. |
| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. |
| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. |
| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. |
| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. |
| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. |
| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. |
| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. |
### Environment variables
In addition to the Rake tasks, there are some environment variables that can be used to modify the process:
| Environment Variable | Data Type | What it does |
| -------------------- |:---------:| ---------------------------------------------------------------------------- |
| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). |
| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. |
| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. |
### Indexing a range of projects or a specific project
Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing.
```shell
root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100
```
Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project
by setting both to the same project ID:
```shell
root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5
Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)...
I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done!
```
## Advanced Search index scopes
When performing a search, the GitLab index uses the following scopes:
| Scope Name | What it searches |
| ---------------- | ---------------------- |
| `commits` | Commit data |
| `projects` | Project data (default) |
| `blobs` | Code |
| `issues` | Issue data |
| `merge_requests` | Merge request data |
| `milestones` | Milestone data |
| `notes` | Note data |
| `snippets` | Snippet data |
| `wiki_blobs` | Wiki contents |
## Tuning
### Guidance on choosing optimal cluster configuration
For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below.
- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand.
- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example).
- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations.
- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details.
- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below.
- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health.
- Number of Elasticsearch shards:
- Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB.
- Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example:
- If you have fewer than about 2,000,000 documents, use the default of 5 shards
- 10,000,000 documents: `10000000/5000000 + 5` = 7 shards
- 100,000,000 documents: `100000000/5000000 + 5` = 25 shards
- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value.
- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations.
### Advanced Search integration settings guidance
- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards.
- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index.
### How to index large instances efficiently
This section may be helpful in the event that the other
[basic instructions](#enable-advanced-search) cause problems
due to large volumes of data being indexed.
WARNING:
Indexing a large instance generates a lot of Sidekiq jobs.
Make sure to prepare for this task by having a [Scalable and Highly Available
Setup](../administration/reference_architectures/index.md) or creating [extra
Sidekiq processes](../administration/operations/extra_sidekiq_processes.md).
1. [Configure your Elasticsearch host and port](#enable-advanced-search).
1. Create empty indices:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:create_empty_index
# Installations from source
bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production
```
1. If this is a re-index of your GitLab instance, clear the index status:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:clear_index_status
# Installations from source
bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production
```
1. [Enable **Elasticsearch indexing**](#enable-advanced-search).
1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed):
- You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search.
- You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete.
In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings.
NOTE:
This step is optional but may help significantly speed up large indexing operations.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"index" : {
"refresh_interval" : "-1",
"number_of_replicas" : 0
} }'
```
1. Index projects and their associated data:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects
# Installations from source
bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production
```
This enqueues a Sidekiq job for each project that needs to be indexed.
You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab**
and select `elastic_commit_indexer`, or you can query indexing status using a Rake task:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects_status
# Installations from source
bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production
Indexing is 65.55% complete (6555/10000 projects)
```
If you want to limit the index to a range of projects you can provide the
`ID_FROM` and `ID_TO` parameters:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000
# Installations from source
bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production
```
Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional.
The above example will index all projects from ID `1001` up to (and including) ID `2000`.
NOTE:
Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects`
can get interrupted. This may happen for many reasons, but it's always safe
to run the indexing task again. It will skip repositories that have
already been indexed.
As the indexer stores the last commit SHA of every indexed repository in the
database, you can run the indexer with the special parameter `UPDATE_INDEX` and
it will check every project repository again to make sure that every commit in
a repository is indexed, which can be useful in case if your index is outdated:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000
# Installations from source
bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production
```
You can also use the `gitlab:elastic:clear_index_status` Rake task to force the
indexer to "forget" all progress, so it retries the indexing process from the
start.
1. Personal snippets are not associated with a project and need to be indexed separately:
```shell
# Omnibus installations
sudo gitlab-rake gitlab:elastic:index_snippets
# Installations from source
bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production
```
1. Enable replication and refreshing again after indexing (only if you previously disabled it):
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"index" : {
"number_of_replicas" : 1,
"refresh_interval" : "1s"
} }'
```
A force merge should be called after enabling the refreshing above.
For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge:
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"settings": {
"index.blocks.write": true
} }'
```
Then, initiate the force merge:
```shell
curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5'
```
After this, if your index is in read-only mode, switch back to read-write:
```shell
curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \
--data '{
"settings": {
"index.blocks.write": false
} }'
```
1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search).
### Deleted documents
Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index.
In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_
However, some larger installations may wish to tune the merge policy settings:
- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \
--data '{
"index" : {
"merge.policy.max_merged_segment": "2gb"
}
}'
```
- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs.
```shell
curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \
--data '{
"index" : {
"merge.policy.reclaim_deletes_weight": "3.0"
}
}'
```
- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues.
## Index large instances with dedicated Sidekiq nodes or processes
Indexing a large instance can be a lengthy and resource-intensive process that has the potential
of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and
availability.
As GitLab allows you to start multiple Sidekiq processes, you can create an
additional process dedicated to indexing a set of queues (or queue group). This way, you can
ensure that indexing queues always have a dedicated worker, while the rest of the queues have
another dedicated worker to avoid contention.
For this purpose, use the [queue selector](../administration/operations/extra_sidekiq_processes.md#queue-selector)
option that allows a more general selection of queue groups using a [worker matching query](../administration/operations/extra_sidekiq_routing.md#worker-matching-query).
To handle these two queue groups, we generally recommend one of the following two options. You can either:
- [Use two queue groups on one single node](#single-node-two-processes).
- [Use two queue groups, one on each node](#two-nodes-one-process-for-each).
For the steps below, consider:
- `feature_category=global_search` as an indexing queue group with its own Sidekiq process.
- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process.
### Single node, two processes
To create both an indexing and a non-indexing Sidekiq process in one node:
1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category=global_search",
"feature_category!=global_search"
]
```
1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md)
for the changes to take effect.
WARNING:
When starting multiple processes, the number of processes cannot exceed the number of CPU
cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject
to the available workload and concurrency settings. For more details, see how to
[run multiple Sidekiq processes](../administration/operations/extra_sidekiq_processes.md).
### Two nodes, one process for each
To handle these queue groups on two nodes:
1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category=global_search"
]
```
1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md)
for the changes to take effect.
1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to:
```ruby
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category!=global_search"
]
```
to set up a non-indexing Sidekiq process.
1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md)
for the changes to take effect.
## Reverting to Basic Search
Sometimes there may be issues with your Elasticsearch index data and as such
GitLab allows you to revert to "basic search" when there are no search
results and assuming that basic search is supported in that scope. This "basic
search" behaves as though you don't have Advanced Search enabled at all for
your instance and search using other data sources (such as PostgreSQL data and Git
data).
## Data recovery: Elasticsearch is a secondary data store only
The use of Elasticsearch in GitLab is only ever as a secondary data store.
This means that all of the data stored in Elasticsearch can always be derived
again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if
the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch.
<!-- This redirect file can be deleted after <2022-09-13>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

2
doc/integration/index.md
View File

@ -60,7 +60,7 @@ GitLab can be integrated with the following enhancements:
or [Kroki](../administration/integration/kroki.md) to use diagrams in AsciiDoc and Markdown documents.
- Attach merge requests to [Trello](trello_power_up.md) cards.
- Enable integrated code intelligence powered by [Sourcegraph](sourcegraph.md).
- Add [Elasticsearch](elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md).
- Add [Elasticsearch](advanced_search/elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md).
## Integrations

11
doc/integration/jira/index.md
View File

@ -67,10 +67,13 @@ The authentication method in Jira depends on whether you host Jira on your own s
## Privacy considerations
If you integrate a private GitLab project with Jira using the [**Jira integration**](#jira-integration),
actions in GitLab issues and merge requests linked to a Jira issue leak information
about the private project to non-administrator Jira users. If your installation uses Jira Cloud,
you can use the [GitLab.com for Jira Cloud app](connect-app.md) to avoid this risk.
All Jira integrations share data with Jira to make it visible outside of GitLab.
If you integrate a private GitLab project with Jira, the private data is
shared with users who have access to your Jira project.
The [**Jira project integration**](#jira-integration) posts GitLab data in the form of comments in Jira issues.
The GitLab.com for Jira Cloud app and Jira DVCS connector share this data through the [**Jira Development Panel**](development_panel.md).
This method provides more fine-grained access control because access can be restricted to certain user groups or roles.
## Third-party Jira integrations

2
doc/raketasks/index.md
View File

@ -26,7 +26,7 @@ The following Rake tasks are available for use with GitLab:
| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. |
| [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. |
| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. |
| [Elasticsearch](../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. |
| [Elasticsearch](../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. |
| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. |
| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. |
| [Geo maintenance](../administration/raketasks/geo.md) | [Geo](../administration/geo/index.md)-related maintenance. |

2
doc/subscriptions/bronze_starter.md
View File

@ -129,7 +129,7 @@ Bronze-level subscribers:
- [Group iterations API](../api/group_iterations.md)
- Project milestones API: [Get all burndown chart events for a single milestone](../api/milestones.md#get-all-burndown-chart-events-for-a-single-milestone)
- [Project iterations API](../api/iterations.md)
- Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/elasticsearch.md) users
- Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/advanced_search/elasticsearch.md) users
- Fields in the [Merge requests API](../api/merge_requests.md) for [merge request approvals](../user/project/merge_requests/approvals/index.md)
- Fields in the [Protected branches API](../api/protected_branches.md) that specify users or groups allowed to merge
- [Merge request approvals API](../api/merge_request_approvals.md)

12
doc/update/index.md
View File

@ -311,9 +311,9 @@ To address the above two scenarios, it is advised to do the following prior to u
## Checking for pending Advanced Search migrations **(PREMIUM SELF)**
This section is only applicable if you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**.
This section is only applicable if you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**.
Major releases require all [Advanced Search migrations](../integration/elasticsearch.md#advanced-search-migrations)
Major releases require all [Advanced Search migrations](../integration/advanced_search/elasticsearch.md#advanced-search-migrations)
to be finished from the most recent minor release in your current version
before the major version upgrade. You can find pending migrations by
running the following command:
@ -333,11 +333,11 @@ sudo -u git -H bundle exec rake gitlab:elastic:list_pending_migrations
### What do I do if my Advanced Search migrations are stuck?
See [how to retry a halted migration](../integration/elasticsearch.md#retry-a-halted-migration).
See [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration).
### What do I do for the error `Elasticsearch version not compatible`
Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/elasticsearch.md#version-requirements).
Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/advanced_search/elasticsearch.md#version-requirements).
## Upgrading without downtime
@ -360,7 +360,7 @@ A *major* upgrade requires the following steps:
It's also important to ensure that any [background migrations have been fully completed](#checking-for-background-migrations-before-upgrading)
before upgrading to a new major version.
If you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**, then
If you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**, then
[ensure all Advanced Search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version within
your current version
before proceeding with the major version upgrade.
@ -466,7 +466,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap
### 15.0.0
- Elasticsearch 6.8 [is no longer supported](../integration/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version).
- Elasticsearch 6.8 [is no longer supported](../integration/advanced_search/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/advanced_search/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version).
- If you run external PostgreSQL, particularly AWS RDS,
[check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue)
to avoid the database crashing.

2
doc/update/patch_versions.md
View File

@ -107,7 +107,7 @@ sudo -u git -H make
### 8. Install/Update `gitlab-elasticsearch-indexer` **(PREMIUM SELF)**
Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch).
Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch).
### 9. Start application

2
doc/update/plan_your_upgrade.md
View File

@ -170,7 +170,7 @@ After updating GitLab, upgrade your runners to match
#### Elasticsearch
After updating GitLab, you may have to upgrade
[Elasticsearch if the new version breaks compatibility](../integration/elasticsearch.md#version-requirements).
[Elasticsearch if the new version breaks compatibility](../integration/advanced_search/elasticsearch.md#version-requirements).
Updating Elasticsearch is **out of scope for GitLab Support**.
## Troubleshooting

2
doc/update/upgrading_from_ce_to_ee.md
View File

@ -88,7 +88,7 @@ sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production
### 4. Install `gitlab-elasticsearch-indexer` **(PREMIUM SELF)**
Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch).
Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch).
### 5. Start application

2
doc/user/admin_area/settings/index.md
View File

@ -79,7 +79,7 @@ The **Geo** setting contains:
The **Integrations** settings contain:
- [Elasticsearch](../../../integration/elasticsearch.md#enable-advanced-search) -
- [Elasticsearch](../../../integration/advanced_search/elasticsearch.md#enable-advanced-search) -
Elasticsearch integration. Elasticsearch AWS IAM.
- [Kroki](../../../administration/integration/kroki.md#enable-kroki-in-gitlab) -
Allow rendering of diagrams in AsciiDoc and Markdown documents using [kroki.io](https://kroki.io).

2
doc/user/application_security/vulnerability_report/index.md
View File

@ -259,7 +259,7 @@ Group, Project, and Security Center Vulnerability Reports. To filter them, use t
> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6345) in GitLab 14.6.
The **Operational vulnerabilities** tab lists vulnerabilities found by the `cluster_image_scanner`.
The **Operational vulnerabilities** tab lists vulnerabilities found by [Operational container scanning](../../clusters/agent/vulnerabilities.md).
This tab appears on the project, group, and Security Center vulnerability reports.
![Operational Vulnerability Tab](img/operational_vulnerability_tab_v14_6.png)

2
doc/user/clusters/agent/vulnerabilities.md
View File

@ -26,6 +26,8 @@ To view vulnerability information in GitLab:
![Cluster agent security tab UI](../img/cluster_agent_security_tab_v14_8.png)
This information can also be found under [operational vulnerabilities](../../../user/application_security/vulnerability_report/index.md#operational-vulnerabilities).
## Enable cluster vulnerability scanning **(ULTIMATE)**
You can use [cluster image scanning](../../application_security/cluster_image_scanning/index.md)

1
doc/user/crm/index.md
View File

@ -177,7 +177,6 @@ API.
FLAG:
On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `contacts_autocomplete`.
On GitLab.com, this feature is available.
This feature is not ready for production use.
When you use the `/add_contacts` or `/remove_contacts` quick actions, follow them with `[contact:` and an autocomplete list appears:

4
doc/user/permissions.md
View File

@ -213,7 +213,7 @@ The following table lists project permissions available for each role:
public and internal projects (not on private projects). [External users](#external-users)
must be given explicit access even if the project is internal. For GitLab.com, see the
[GitLab.com visibility settings](gitlab_com/index.md#visibility-settings).
2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves.
2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to.
3. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [protected branches](project/protected_branches.md).
4. If the [branch is protected](project/protected_branches.md), this depends on the access Developers and Maintainers are given.
5. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see [repository information like commits and release evidence](project/releases/index.md#view-a-release-and-download-assets).
@ -358,7 +358,7 @@ Read through the documentation on [permissions for File Locking](project/file_lo
### Confidential Issues permissions
[Confidential issues](project/issues/confidential_issues.md) can be accessed by users with reporter and higher permission levels,
as well as by guest users that create a confidential issue. To learn more,
as well as by guest users that create a confidential issue or are assigned to it. To learn more,
read through the documentation on [permissions and access to confidential issues](project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues).
### Container Registry visibility permissions

5
doc/user/project/issues/confidential_issues.md
View File

@ -36,7 +36,7 @@ The second way is to locate the **Confidentiality** section in the sidebar and s
| Turn off confidentiality | Turn on confidentiality |
| :-----------: | :----------: |
| ![Turn off confidentiality](img/turn_off_confidentiality_v15_0.png) | ![Turn on confidentiality](img/turn_on_confidentiality_v15_0.png) |
| ![Turn off confidentiality](img/turn_off_confidentiality_v15_1.png) | ![Turn on confidentiality](img/turn_on_confidentiality_v15_1.png) |
Every change from regular to confidential and vice versa, is indicated by a
system note in the issue's comments.
@ -84,6 +84,9 @@ There are two kinds of level access for confidential issues. The general rule
is that confidential issues are visible only to members of a project with at
least the Reporter role. However, a guest user can also create
confidential issues, but can only view the ones that they created themselves.
Users with the Guest role or non-members can also read the confidential issue if they are assigned to the issue.
When a Guest user or non-member is unassigned from a confidential issue,
they can no longer view it.
Confidential issues are also hidden in search results for unprivileged users.
For example, here's what a user with the Maintainer role and the Guest role

BIN
doc/user/project/issues/img/turn_off_confidentiality_v15_1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 20 KiB

BIN
doc/user/project/issues/img/turn_on_confidentiality_v15_1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 37 KiB

21
doc/user/project/repository/push_rules.md
View File

@ -26,6 +26,9 @@ For custom push rules use [server hooks](../../../administration/server_hooks.md
You can create push rules for all new projects to inherit, but they can be overridden
at the project level or the [group level](../../group/index.md#group-push-rules).
All projects created after you configure global push rules inherit this
configuration. However, each existing project must be updated manually, using the
process described in [Override global push rules per project](#override-global-push-rules-per-project).
Prerequisite:
@ -42,7 +45,8 @@ To create global push rules:
## Override global push rules per project
The push rule of an individual project overrides the global push rule.
To override global push rules for a specific project:
To override global push rules for a specific project, or to update the rules
for an existing project to match new global push rules:
1. On the top bar, select **Menu > Projects** and find your project.
1. On the left sidebar, select **Settings > Repository**.
@ -77,10 +81,12 @@ Use these rules for your commit messages.
## Validate branch names
To validate your branch names, enter a regular expression for **Branch name**.
To allow any branch name, leave empty. Your
[default branch](branches/default.md) is always allowed.
To allow any branch name, leave empty. Your [default branch](branches/default.md)
is always allowed. Certain formats of branch names are restricted by default for
security purposes. Names with 40 hexadecimal characters, similar to Git commit hashes,
are prohibited.
Examples:
Some validation examples:
- Branches must start with `JIRA-`.
@ -101,11 +107,6 @@ Examples:
`^[a-z0-9\\-]{4,15}$`
```
NOTE:
In GitLab 12.10 and later, certain formats of branch names are restricted by
default for security purposes. 40-character hexadecimal names, similar to Git
commit hashes, are prohibited.
## Prevent unintended consequences
Use these rules to prevent unintended consequences.
@ -134,6 +135,8 @@ Never commit secrets, such as credential files and SSH private keys, to a versio
system. In GitLab, you can use a predefined list of files to block those files from a
repository. Merge requests that contain a file that matches the list are blocked.
This push rule does not restrict files already committed to the repository.
You must update the configuration of existing projects to use the rule, using the
process described in [Override global push rules per project](#override-global-push-rules-per-project).
Files blocked by this rule are listed below. For a complete list of criteria, refer to
[`files_denylist.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml).

2
doc/user/search/advanced_search.md
View File

@ -45,7 +45,7 @@ Advanced Search can be useful in various scenarios:
## Configuring Advanced Search
For self-managed GitLab instances, an administrator must
[configure Advanced Search](../../integration/elasticsearch.md).
[configure Advanced Search](../../integration/advanced_search/elasticsearch.md).
On GitLab.com, Advanced Search is enabled.

43
lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb Normal file
View File

@ -0,0 +1,43 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Backfill projectfeatures.package_registry_access_level depending on projects.packages_enabled
class BackfillProjectFeaturePackageRegistryAccessLevel < ::Gitlab::BackgroundMigration::BatchedMigrationJob
FEATURE_DISABLED = 0 # ProjectFeature::DISABLED
FEATURE_PRIVATE = 10 # ProjectFeature::PRIVATE
FEATURE_ENABLED = 20 # ProjectFeature::ENABLED
FEATURE_PUBLIC = 30 # ProjectFeature::PUBLIC
PROJECT_PRIVATE = 0 # Gitlab::VisibilityLevel::PRIVATE
PROJECT_INTERNAL = 10 # Gitlab::VisibilityLevel::INTERNAL
PROJECT_PUBLIC = 20 # Gitlab::VisibilityLevel::PUBLIC
# Migration only version of ProjectFeature table
class ProjectFeature < ::ApplicationRecord
self.table_name = 'project_features'
end
def perform
each_sub_batch(operation_name: :update_all) do |sub_batch|
ProjectFeature.connection.execute(
<<~SQL
UPDATE project_features pf
SET package_registry_access_level = (CASE p.packages_enabled
WHEN true THEN (CASE p.visibility_level
WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC}
WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED}
WHEN #{PROJECT_PRIVATE} THEN #{FEATURE_PRIVATE}
END)
WHEN false THEN #{FEATURE_DISABLED}
ELSE #{FEATURE_DISABLED}
END)
FROM projects p
WHERE pf.project_id = p.id AND
pf.project_id BETWEEN #{start_id} AND #{end_id}
SQL
)
end
end
end
end
end

23
locale/gitlab.pot
View File

@ -13640,9 +13640,6 @@ msgstr ""
msgid "Edit"
msgstr ""
msgid "Edit %{issuable}"
msgstr ""
msgid "Edit %{name}"
msgstr ""
@ -18318,16 +18315,16 @@ msgstr ""
msgid "GroupSettings|Projects will be permanently deleted after a %{waiting_period}-day delay. This delay can be %{link_start}customized by an admin%{link_end} in instance settings. Inherited by subgroups."
msgstr ""
msgid "GroupSettings|Select a project with the %{code_start}.gitlab/insights.yml%{code_end} file"
msgstr ""
msgid "GroupSettings|Select a subgroup to use as the source for custom project templates for this group."
msgstr ""
msgid "GroupSettings|Select parent group"
msgstr ""
msgid "GroupSettings|Select the project that contains your custom Insights file."
msgid "GroupSettings|Select the project containing the %{code_start}.gitlab/insights.yml%{code_end} file"
msgstr ""
msgid "GroupSettings|Select the project containing your custom Insights file. %{help_link_start}What is Insights?%{help_link_end}"
msgstr ""
msgid "GroupSettings|Set a size limit for all content in each Pages site in this group. %{link_start}Learn more.%{link_end}"
@ -26643,7 +26640,7 @@ msgstr ""
msgid "One or more of your personal access tokens will expire in %{days_to_expire} days or less:"
msgstr ""
msgid "Only %{workspaceType} members with at least Reporter role can view or be notified about this %{issuableType}."
msgid "Only %{workspaceType} members with %{permissions} can view or be notified about this %{issuableType}."
msgstr ""
msgid "Only 'Reporter' roles and above on tiers Premium and above can see Value Stream Analytics."
@ -26673,7 +26670,7 @@ msgstr ""
msgid "Only effective when remote storage is enabled. Set to 0 for no size limit."
msgstr ""
msgid "Only group members with at least Reporter role can view or be notified about this epic."
msgid "Only group members with at least the Reporter role can view or be notified about this epic"
msgstr ""
msgid "Only include features new to your current subscription tier."
@ -43375,7 +43372,7 @@ msgstr ""
msgid "You are going to turn off the confidentiality. This means %{strongStart}everyone%{strongEnd} will be able to see and leave a comment on this %{issuableType}."
msgstr ""
msgid "You are going to turn on confidentiality. Only %{context} members with %{strongStart}at least Reporter role%{strongEnd} can view or be notified about this %{issuableType}."
msgid "You are going to turn on confidentiality. Only %{context} members with %{strongStart}%{permissions}%{strongEnd} can view or be notified about this %{issuableType}."
msgstr ""
msgid "You are not allowed to %{action} a user"
@ -44444,6 +44441,12 @@ msgstr ""
msgid "at"
msgstr ""
msgid "at least the Reporter role"
msgstr ""
msgid "at least the Reporter role, the author, and assignees"
msgstr ""
msgid "at risk"
msgstr ""

15
spec/fixtures/glfm/example_snapshots/html.yml vendored
View File

@ -1909,7 +1909,18 @@
]]>
<p>okay</p>
static: |2-
&lt;![CDATA[
function matchwo(a,b)
{
if (a &lt; b &amp;&amp; a &lt; 0) then {
return 1;
} else {
return 0;
}
}
]]&gt;
<p data-sourcepos="13:1-13:4" dir="auto">okay</p>
wysiwyg: |-
Error - check implementation:
@ -7233,7 +7244,7 @@
canonical: |
<p>foo <!ELEMENT br EMPTY></p>
static: |-
<p data-sourcepos="1:1-1:23" dir="auto">foo </p>
<p data-sourcepos="1:1-1:23" dir="auto">foo &lt;!ELEMENT br EMPTY&gt;</p>
wysiwyg: |-
Error - check implementation:
Cannot destructure property 'className' of 'hastNode.properties' as it is undefined.
@ -7241,7 +7252,7 @@
canonical: |
<p>foo <![CDATA[>&<]]></p>
static: |-
<p data-sourcepos="1:1-1:19" dir="auto">foo &amp;</p>
<p data-sourcepos="1:1-1:19" dir="auto">foo &lt;![CDATA[&gt;&amp;&lt;]]&gt;</p>
wysiwyg: |-
Error - check implementation:
Cannot destructure property 'className' of 'hastNode.properties' as it is undefined.

4
spec/frontend/sidebar/components/confidential/sidebar_confidentiality_content_spec.js
View File

@ -69,14 +69,14 @@ describe('Sidebar Confidentiality Content', () => {
variant: 'warning',
});
expect(alertEl.text()).toBe(
'Only project members with at least Reporter role can view or be notified about this issue.',
'Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.',
);
});
it('displays a correct confidential text for epic', () => {
createComponent({ confidential: true, issuableType: 'epic' });
expect(findText().findComponent(GlAlert).text()).toBe(
'Only group members with at least Reporter role can view or be notified about this epic.',
'Only group members with at least the Reporter role can view or be notified about this epic.',
);
});
});

2
spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js
View File

@ -89,7 +89,7 @@ describe('Sidebar Confidentiality Form', () => {
it('renders a message about making an issue confidential', () => {
expect(findWarningMessage().text()).toBe(
'You are going to turn on confidentiality. Only project members with at least Reporter role can view or be notified about this issue.',
'You are going to turn on confidentiality. Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.',
);
});

4
spec/frontend/vue_shared/components/confidentiality_badge_spec.js
View File

@ -29,8 +29,8 @@ describe('ConfidentialityBadge', () => {
it.each`
workspaceType | issuableType | expectedTooltip
${WorkspaceType.project} | ${IssuableType.Issue} | ${'Only project members with at least Reporter role can view or be notified about this issue.'}
${WorkspaceType.group} | ${IssuableType.Epic} | ${'Only group members with at least Reporter role can view or be notified about this epic.'}
${WorkspaceType.project} | ${IssuableType.Issue} | ${'Only project members with at least the Reporter role, the author, and assignees can view or be notified about this issue.'}
${WorkspaceType.group} | ${IssuableType.Epic} | ${'Only group members with at least the Reporter role can view or be notified about this epic.'}
`(
'should render gl-badge with correct tooltip when workspaceType is $workspaceType and issuableType is $issuableType',
({ workspaceType, issuableType, expectedTooltip }) => {

124
spec/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level_spec.rb Normal file
View File

@ -0,0 +1,124 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFeaturePackageRegistryAccessLevel do
let(:non_null_project_features) { { pages_access_level: 20 } }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:project_features) { table(:project_features) }
let(:namespace1) { namespaces.create!(name: 'namespace 1', path: 'namespace1') }
let(:namespace2) { namespaces.create!(name: 'namespace 2', path: 'namespace2') }
let(:namespace3) { namespaces.create!(name: 'namespace 3', path: 'namespace3') }
let(:namespace4) { namespaces.create!(name: 'namespace 4', path: 'namespace4') }
let(:namespace5) { namespaces.create!(name: 'namespace 5', path: 'namespace5') }
let(:namespace6) { namespaces.create!(name: 'namespace 6', path: 'namespace6') }
let(:project1) do
projects.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id, packages_enabled: false)
end
let(:project2) do
projects.create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id, packages_enabled: nil)
end
let(:project3) do
projects.create!(
namespace_id: namespace3.id,
project_namespace_id: namespace3.id,
packages_enabled: true,
visibility_level: Gitlab::VisibilityLevel::PRIVATE
)
end
let(:project4) do
projects.create!(
namespace_id: namespace4.id,
project_namespace_id: namespace4.id,
packages_enabled: true, visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
let(:project5) do
projects.create!(
namespace_id: namespace5.id,
project_namespace_id: namespace5.id,
packages_enabled: true,
visibility_level: Gitlab::VisibilityLevel::PUBLIC
)
end
let(:project6) do
projects.create!(namespace_id: namespace6.id, project_namespace_id: namespace6.id, packages_enabled: false)
end
let!(:project_feature1) do
project_features.create!(
project_id: project1.id,
package_registry_access_level: ProjectFeature::ENABLED,
**non_null_project_features
)
end
let!(:project_feature2) do
project_features.create!(
project_id: project2.id,
package_registry_access_level: ProjectFeature::ENABLED,
**non_null_project_features
)
end
let!(:project_feature3) do
project_features.create!(
project_id: project3.id,
package_registry_access_level: ProjectFeature::DISABLED,
**non_null_project_features
)
end
let!(:project_feature4) do
project_features.create!(
project_id: project4.id,
package_registry_access_level: ProjectFeature::DISABLED,
**non_null_project_features
)
end
let!(:project_feature5) do
project_features.create!(
project_id: project5.id,
package_registry_access_level: ProjectFeature::DISABLED,
**non_null_project_features
)
end
let!(:project_feature6) do
project_features.create!(
project_id: project6.id,
package_registry_access_level: ProjectFeature::ENABLED,
**non_null_project_features
)
end
subject(:perform_migration) do
described_class.new(start_id: project1.id,
end_id: project5.id,
batch_table: :projects,
batch_column: :id,
sub_batch_size: 2,
pause_ms: 0,
connection: ActiveRecord::Base.connection)
.perform
end
it 'backfills project_features.package_registry_access_level', :aggregate_failures do
perform_migration
expect(project_feature1.reload.package_registry_access_level).to eq(ProjectFeature::DISABLED)
expect(project_feature2.reload.package_registry_access_level).to eq(ProjectFeature::DISABLED)
expect(project_feature3.reload.package_registry_access_level).to eq(ProjectFeature::PRIVATE)
expect(project_feature4.reload.package_registry_access_level).to eq(ProjectFeature::ENABLED)
expect(project_feature5.reload.package_registry_access_level).to eq(ProjectFeature::PUBLIC)
expect(project_feature6.reload.package_registry_access_level).to eq(ProjectFeature::ENABLED)
end
end

24
spec/migrations/queue_backfill_project_feature_package_registry_access_level_spec.rb Normal file
View File

@ -0,0 +1,24 @@
# frozen_string_literal: true
require 'spec_helper'
require_migration!
RSpec.describe QueueBackfillProjectFeaturePackageRegistryAccessLevel do
let_it_be(:batched_migration) { described_class::MIGRATION }
it 'schedules a new batched migration' do
reversible_migration do |migration|
migration.before -> {
expect(batched_migration).not_to have_scheduled_batched_migration
}
migration.after -> {
expect(batched_migration).to have_scheduled_batched_migration(
table_name: :projects,
column_name: :id,
interval: described_class::DELAY_INTERVAL
)
}
end
end
end

5
spec/support/shared_examples/features/sidebar_shared_examples.rb
View File

@ -109,9 +109,8 @@ RSpec.shared_examples 'issue boards sidebar' do
wait_for_requests
expect(page).to have_content(
_('Only project members with at least' \
' Reporter role can view or be' \
' notified about this issue.')
_('Only project members with at least the Reporter role, the author, and assignees' \
' can view or be notified about this issue.')
)
end
end

2
spec/views/projects/issues/_service_desk_info_content.html.haml_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe 'projects/issues/_service_desk_info_content' do
RSpec.describe 'projects/issues/service_desk/_service_desk_info_content' do
let_it_be(:project) { create(:project) }
let_it_be(:user) { create(:user) }
let_it_be(:service_desk_address) { 'address@example.com' }